2024-12-11 06:14:11,511 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-11 06:14:11,525 main DEBUG Took 0.012185 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 06:14:11,526 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 06:14:11,526 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 06:14:11,528 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 06:14:11,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,538 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 06:14:11,551 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,558 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,560 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,561 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,562 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,563 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,564 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,566 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 06:14:11,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,567 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 06:14:11,569 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 06:14:11,571 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 06:14:11,573 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 06:14:11,574 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 06:14:11,575 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 06:14:11,576 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 06:14:11,585 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 06:14:11,587 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 06:14:11,589 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 06:14:11,589 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 06:14:11,589 main DEBUG createAppenders(={Console}) 2024-12-11 06:14:11,590 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-11 06:14:11,590 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-11 06:14:11,590 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-11 06:14:11,591 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 06:14:11,591 main DEBUG OutputStream closed 2024-12-11 06:14:11,591 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 06:14:11,591 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 06:14:11,592 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-11 06:14:11,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 06:14:11,655 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 06:14:11,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 06:14:11,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 06:14:11,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 06:14:11,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 06:14:11,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 06:14:11,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 06:14:11,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 06:14:11,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 06:14:11,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 06:14:11,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 06:14:11,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 06:14:11,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 06:14:11,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 06:14:11,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 06:14:11,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 06:14:11,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 06:14:11,664 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 06:14:11,664 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-11 06:14:11,664 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 06:14:11,665 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-11T06:14:11,892 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11 2024-12-11 06:14:11,894 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 06:14:11,895 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T06:14:11,902 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-11T06:14:11,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T06:14:11,938 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920, deleteOnExit=true 2024-12-11T06:14:11,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T06:14:11,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/test.cache.data in system properties and HBase conf 2024-12-11T06:14:11,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T06:14:11,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.log.dir in system properties and HBase conf 2024-12-11T06:14:11,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T06:14:11,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T06:14:11,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T06:14:12,053 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T06:14:12,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T06:14:12,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T06:14:12,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T06:14:12,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T06:14:12,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T06:14:12,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T06:14:12,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T06:14:12,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T06:14:12,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T06:14:12,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T06:14:12,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/nfs.dump.dir in system properties and HBase conf 2024-12-11T06:14:12,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/java.io.tmpdir in system properties and HBase conf 2024-12-11T06:14:12,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T06:14:12,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T06:14:12,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T06:14:13,466 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T06:14:13,568 INFO [Time-limited test {}] log.Log(170): Logging initialized @2723ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T06:14:13,661 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T06:14:13,756 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T06:14:13,838 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T06:14:13,838 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T06:14:13,840 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T06:14:13,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T06:14:13,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.log.dir/,AVAILABLE} 2024-12-11T06:14:13,919 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T06:14:14,181 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/java.io.tmpdir/jetty-localhost-33647-hadoop-hdfs-3_4_1-tests_jar-_-any-8231044823399590690/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T06:14:14,188 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:33647} 2024-12-11T06:14:14,189 INFO [Time-limited test {}] server.Server(415): Started @3345ms 2024-12-11T06:14:14,858 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T06:14:14,867 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T06:14:14,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T06:14:14,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T06:14:14,869 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T06:14:14,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.log.dir/,AVAILABLE} 2024-12-11T06:14:14,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T06:14:15,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/java.io.tmpdir/jetty-localhost-46311-hadoop-hdfs-3_4_1-tests_jar-_-any-14202625388299801349/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T06:14:15,025 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:46311} 2024-12-11T06:14:15,025 INFO [Time-limited test {}] server.Server(415): Started @4181ms 2024-12-11T06:14:15,094 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T06:14:16,511 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data2/current/BP-461083816-172.17.0.3-1733897652939/current, will proceed with Du for space computation calculation, 2024-12-11T06:14:16,511 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data1/current/BP-461083816-172.17.0.3-1733897652939/current, will proceed with Du for space computation calculation, 2024-12-11T06:14:16,551 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T06:14:16,605 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8fecb3cc4579c167 with lease ID 0x6554fce9d19ce510: Processing first storage report for DS-73cb111b-9445-42c1-a84f-533c2b23174b from datanode DatanodeRegistration(127.0.0.1:33105, datanodeUuid=24e1446c-b24e-40c8-80d4-824be2c4b671, infoPort=41335, infoSecurePort=0, ipcPort=35345, storageInfo=lv=-57;cid=testClusterID;nsid=406686619;c=1733897652939) 2024-12-11T06:14:16,607 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8fecb3cc4579c167 with lease ID 0x6554fce9d19ce510: from storage DS-73cb111b-9445-42c1-a84f-533c2b23174b node DatanodeRegistration(127.0.0.1:33105, datanodeUuid=24e1446c-b24e-40c8-80d4-824be2c4b671, infoPort=41335, infoSecurePort=0, ipcPort=35345, storageInfo=lv=-57;cid=testClusterID;nsid=406686619;c=1733897652939), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-11T06:14:16,607 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8fecb3cc4579c167 with lease ID 0x6554fce9d19ce510: Processing first storage report for DS-4faaa0f3-f001-434b-80db-efea0bd950cc from datanode DatanodeRegistration(127.0.0.1:33105, datanodeUuid=24e1446c-b24e-40c8-80d4-824be2c4b671, infoPort=41335, infoSecurePort=0, ipcPort=35345, storageInfo=lv=-57;cid=testClusterID;nsid=406686619;c=1733897652939) 2024-12-11T06:14:16,607 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8fecb3cc4579c167 with lease ID 0x6554fce9d19ce510: from storage DS-4faaa0f3-f001-434b-80db-efea0bd950cc node DatanodeRegistration(127.0.0.1:33105, datanodeUuid=24e1446c-b24e-40c8-80d4-824be2c4b671, infoPort=41335, infoSecurePort=0, ipcPort=35345, storageInfo=lv=-57;cid=testClusterID;nsid=406686619;c=1733897652939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T06:14:16,608 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11 2024-12-11T06:14:16,711 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/zookeeper_0, clientPort=60143, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T06:14:16,729 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60143 2024-12-11T06:14:16,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:16,747 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:17,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741825_1001 (size=7) 2024-12-11T06:14:17,476 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 with version=8 2024-12-11T06:14:17,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/hbase-staging 2024-12-11T06:14:17,589 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T06:14:17,898 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4518703b2c1d:0 server-side Connection retries=45 2024-12-11T06:14:17,909 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:17,909 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:17,915 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T06:14:17,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:17,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T06:14:18,077 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T06:14:18,154 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T06:14:18,166 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T06:14:18,170 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T06:14:18,199 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4505 (auto-detected) 2024-12-11T06:14:18,200 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-11T06:14:18,235 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41397 2024-12-11T06:14:18,267 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41397 connecting to ZooKeeper ensemble=127.0.0.1:60143 2024-12-11T06:14:18,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413970x0, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T06:14:18,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41397-0x1001382a0340000 connected 2024-12-11T06:14:18,525 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:18,528 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:18,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T06:14:18,573 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915, hbase.cluster.distributed=false 2024-12-11T06:14:18,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T06:14:18,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41397 2024-12-11T06:14:18,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41397 2024-12-11T06:14:18,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41397 2024-12-11T06:14:18,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41397 2024-12-11T06:14:18,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41397 2024-12-11T06:14:18,888 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4518703b2c1d:0 server-side Connection retries=45 2024-12-11T06:14:18,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:18,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:18,893 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T06:14:18,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T06:14:18,893 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T06:14:18,897 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T06:14:18,903 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T06:14:18,910 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35425 2024-12-11T06:14:18,912 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35425 connecting to ZooKeeper ensemble=127.0.0.1:60143 2024-12-11T06:14:18,916 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:18,921 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:19,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354250x0, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T06:14:19,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354250x0, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T06:14:19,029 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T06:14:19,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35425-0x1001382a0340001 connected 2024-12-11T06:14:19,116 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T06:14:19,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T06:14:19,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T06:14:19,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35425 2024-12-11T06:14:19,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35425 2024-12-11T06:14:19,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35425 2024-12-11T06:14:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35425 2024-12-11T06:14:19,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35425 2024-12-11T06:14:19,196 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4518703b2c1d:41397 2024-12-11T06:14:19,197 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4518703b2c1d,41397,1733897657727 2024-12-11T06:14:19,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T06:14:19,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T06:14:19,219 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4518703b2c1d,41397,1733897657727 2024-12-11T06:14:19,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T06:14:19,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:19,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:19,254 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T06:14:19,255 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4518703b2c1d,41397,1733897657727 from backup master directory 2024-12-11T06:14:19,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4518703b2c1d,41397,1733897657727 2024-12-11T06:14:19,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T06:14:19,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T06:14:19,271 WARN [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T06:14:19,271 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4518703b2c1d,41397,1733897657727 2024-12-11T06:14:19,273 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T06:14:19,275 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T06:14:19,336 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/hbase.id] with ID: a7644275-0103-4d98-b05f-d576422ffe05 2024-12-11T06:14:19,337 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/.tmp/hbase.id 2024-12-11T06:14:19,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741826_1002 (size=42) 2024-12-11T06:14:19,748 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/.tmp/hbase.id]:[hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/hbase.id] 2024-12-11T06:14:19,801 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T06:14:19,807 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T06:14:19,831 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-11T06:14:19,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:19,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:19,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741827_1003 (size=196) 2024-12-11T06:14:19,887 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:14:19,889 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T06:14:19,907 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:19,911 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T06:14:19,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741828_1004 (size=1189) 2024-12-11T06:14:20,381 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store 2024-12-11T06:14:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741829_1005 (size=34) 2024-12-11T06:14:20,413 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T06:14:20,431 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:20,432 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T06:14:20,433 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:14:20,433 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:14:20,435 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T06:14:20,435 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:14:20,436 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:14:20,437 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733897660432Disabling compacts and flushes for region at 1733897660432Disabling writes for close at 1733897660435 (+3 ms)Writing region close event to WAL at 1733897660435Closed at 1733897660435 2024-12-11T06:14:20,440 WARN [master/4518703b2c1d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/.initializing 2024-12-11T06:14:20,440 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/WALs/4518703b2c1d,41397,1733897657727 2024-12-11T06:14:20,451 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T06:14:20,468 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4518703b2c1d%2C41397%2C1733897657727, suffix=, logDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/WALs/4518703b2c1d,41397,1733897657727, archiveDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/oldWALs, maxLogs=10 2024-12-11T06:14:20,492 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/WALs/4518703b2c1d,41397,1733897657727/4518703b2c1d%2C41397%2C1733897657727.1733897660473, exclude list is [], retry=0 2024-12-11T06:14:20,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33105,DS-73cb111b-9445-42c1-a84f-533c2b23174b,DISK] 2024-12-11T06:14:20,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T06:14:20,551 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/WALs/4518703b2c1d,41397,1733897657727/4518703b2c1d%2C41397%2C1733897657727.1733897660473 2024-12-11T06:14:20,552 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-12-11T06:14:20,553 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:14:20,553 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:20,557 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,559 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T06:14:20,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:20,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:20,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T06:14:20,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:20,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:20,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T06:14:20,654 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:20,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:20,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T06:14:20,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:20,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:20,669 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,676 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,678 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,687 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,688 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,692 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T06:14:20,697 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T06:14:20,702 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:14:20,703 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59706789, jitterRate=-0.11029951274394989}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T06:14:20,710 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733897660576Initializing all the Stores at 1733897660579 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897660579Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897660580 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897660580Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897660580Cleaning up temporary data from old regions at 1733897660688 (+108 ms)Region opened successfully at 1733897660710 (+22 ms) 2024-12-11T06:14:20,712 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T06:14:20,746 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cbd7d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4518703b2c1d/172.17.0.3:0 2024-12-11T06:14:20,777 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T06:14:20,787 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T06:14:20,788 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T06:14:20,790 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T06:14:20,792 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T06:14:20,797 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T06:14:20,797 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T06:14:20,827 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T06:14:20,840 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T06:14:20,890 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T06:14:20,893 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T06:14:20,895 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T06:14:21,005 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T06:14:21,007 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T06:14:21,013 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T06:14:21,078 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T06:14:21,080 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T06:14:21,139 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T06:14:21,164 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T06:14:21,174 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T06:14:21,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T06:14:21,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T06:14:21,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,189 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4518703b2c1d,41397,1733897657727, sessionid=0x1001382a0340000, setting cluster-up flag (Was=false) 2024-12-11T06:14:21,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,248 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T06:14:21,251 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4518703b2c1d,41397,1733897657727 2024-12-11T06:14:21,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:21,356 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T06:14:21,358 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4518703b2c1d,41397,1733897657727 2024-12-11T06:14:21,364 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T06:14:21,381 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(746): ClusterId : a7644275-0103-4d98-b05f-d576422ffe05 2024-12-11T06:14:21,383 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T06:14:21,443 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T06:14:21,443 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T06:14:21,445 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T06:14:21,455 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T06:14:21,462 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T06:14:21,468 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4518703b2c1d,41397,1733897657727 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T06:14:21,482 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T06:14:21,483 DEBUG [RS:0;4518703b2c1d:35425 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201b05df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4518703b2c1d/172.17.0.3:0 2024-12-11T06:14:21,484 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4518703b2c1d:0, corePoolSize=5, maxPoolSize=5 2024-12-11T06:14:21,484 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4518703b2c1d:0, corePoolSize=5, maxPoolSize=5 2024-12-11T06:14:21,484 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4518703b2c1d:0, corePoolSize=5, maxPoolSize=5 2024-12-11T06:14:21,484 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4518703b2c1d:0, corePoolSize=5, maxPoolSize=5 2024-12-11T06:14:21,485 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4518703b2c1d:0, corePoolSize=10, maxPoolSize=10 2024-12-11T06:14:21,485 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,485 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4518703b2c1d:0, corePoolSize=2, maxPoolSize=2 2024-12-11T06:14:21,485 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,504 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T06:14:21,504 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T06:14:21,508 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4518703b2c1d:35425 2024-12-11T06:14:21,511 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733897691510 2024-12-11T06:14:21,511 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:21,512 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T06:14:21,512 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T06:14:21,512 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T06:14:21,512 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T06:14:21,513 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T06:14:21,514 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T06:14:21,515 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(2659): reportForDuty to master=4518703b2c1d,41397,1733897657727 with port=35425, startcode=1733897658824 2024-12-11T06:14:21,518 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T06:14:21,518 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T06:14:21,519 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T06:14:21,519 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T06:14:21,519 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,527 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T06:14:21,528 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T06:14:21,529 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T06:14:21,531 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T06:14:21,532 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T06:14:21,532 DEBUG [RS:0;4518703b2c1d:35425 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T06:14:21,535 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.large.0-1733897661533,5,FailOnTimeoutGroup] 2024-12-11T06:14:21,536 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.small.0-1733897661535,5,FailOnTimeoutGroup] 2024-12-11T06:14:21,536 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,536 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T06:14:21,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741831_1007 (size=1321) 2024-12-11T06:14:21,538 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,538 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T06:14:21,580 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41397 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,583 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41397 {}] master.ServerManager(517): Registering regionserver=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,603 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:14:21,603 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33311 2024-12-11T06:14:21,603 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T06:14:21,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T06:14:21,694 DEBUG [RS:0;4518703b2c1d:35425 {}] zookeeper.ZKUtil(111): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,695 WARN [RS:0;4518703b2c1d:35425 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T06:14:21,695 INFO [RS:0;4518703b2c1d:35425 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T06:14:21,695 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,697 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4518703b2c1d,35425,1733897658824] 2024-12-11T06:14:21,725 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T06:14:21,741 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T06:14:21,748 INFO [RS:0;4518703b2c1d:35425 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T06:14:21,748 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,749 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T06:14:21,756 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T06:14:21,758 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,758 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4518703b2c1d:0, corePoolSize=2, maxPoolSize=2 2024-12-11T06:14:21,759 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,760 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,760 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,760 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,760 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,760 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4518703b2c1d:0, corePoolSize=1, maxPoolSize=1 2024-12-11T06:14:21,761 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4518703b2c1d:0, corePoolSize=3, maxPoolSize=3 2024-12-11T06:14:21,761 DEBUG [RS:0;4518703b2c1d:35425 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0, corePoolSize=3, maxPoolSize=3 2024-12-11T06:14:21,763 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,763 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,763 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,764 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,764 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,764 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,35425,1733897658824-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T06:14:21,791 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T06:14:21,794 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,35425,1733897658824-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,794 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,794 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.Replication(171): 4518703b2c1d,35425,1733897658824 started 2024-12-11T06:14:21,820 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:21,820 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1482): Serving as 4518703b2c1d,35425,1733897658824, RpcServer on 4518703b2c1d/172.17.0.3:35425, sessionid=0x1001382a0340001 2024-12-11T06:14:21,821 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T06:14:21,821 DEBUG [RS:0;4518703b2c1d:35425 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,822 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4518703b2c1d,35425,1733897658824' 2024-12-11T06:14:21,822 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T06:14:21,823 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T06:14:21,823 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T06:14:21,823 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T06:14:21,824 DEBUG [RS:0;4518703b2c1d:35425 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4518703b2c1d,35425,1733897658824 2024-12-11T06:14:21,824 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4518703b2c1d,35425,1733897658824' 2024-12-11T06:14:21,824 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T06:14:21,824 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T06:14:21,829 DEBUG [RS:0;4518703b2c1d:35425 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T06:14:21,829 INFO [RS:0;4518703b2c1d:35425 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T06:14:21,829 INFO [RS:0;4518703b2c1d:35425 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T06:14:21,934 INFO [RS:0;4518703b2c1d:35425 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T06:14:21,936 INFO [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4518703b2c1d%2C35425%2C1733897658824, suffix=, logDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824, archiveDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/oldWALs, maxLogs=32 2024-12-11T06:14:21,940 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T06:14:21,940 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:14:21,956 DEBUG [RS:0;4518703b2c1d:35425 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824/4518703b2c1d%2C35425%2C1733897658824.1733897661939, exclude list is [], retry=0 2024-12-11T06:14:21,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741832_1008 (size=32) 2024-12-11T06:14:21,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33105,DS-73cb111b-9445-42c1-a84f-533c2b23174b,DISK] 2024-12-11T06:14:21,969 INFO [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824/4518703b2c1d%2C35425%2C1733897658824.1733897661939 2024-12-11T06:14:21,970 DEBUG [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-12-11T06:14:22,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:22,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T06:14:22,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T06:14:22,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:22,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:22,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T06:14:22,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T06:14:22,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:22,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:22,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T06:14:22,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T06:14:22,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:22,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:22,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T06:14:22,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T06:14:22,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:22,388 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:22,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T06:14:22,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740 2024-12-11T06:14:22,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740 2024-12-11T06:14:22,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T06:14:22,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T06:14:22,394 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:14:22,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T06:14:22,403 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:14:22,405 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66755232, jitterRate=-0.005269527435302734}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:14:22,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733897662363Initializing all the Stores at 1733897662365 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662365Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662365Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897662365Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662365Cleaning up temporary data from old regions at 1733897662393 (+28 ms)Region opened successfully at 1733897662407 (+14 ms) 2024-12-11T06:14:22,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T06:14:22,407 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T06:14:22,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T06:14:22,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T06:14:22,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T06:14:22,409 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T06:14:22,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733897662407Disabling compacts and flushes for region at 1733897662407Disabling writes for close at 1733897662407Writing region close event to WAL at 1733897662409 (+2 ms)Closed at 1733897662409 2024-12-11T06:14:22,414 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T06:14:22,414 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T06:14:22,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T06:14:22,434 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T06:14:22,446 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T06:14:22,599 DEBUG [4518703b2c1d:41397 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-11T06:14:22,615 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:22,622 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4518703b2c1d,35425,1733897658824, state=OPENING 2024-12-11T06:14:22,681 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T06:14:22,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:22,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:14:22,746 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T06:14:22,748 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T06:14:22,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:22,753 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T06:14:22,926 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T06:14:22,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T06:14:22,941 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T06:14:22,941 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T06:14:22,942 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T06:14:22,946 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4518703b2c1d%2C35425%2C1733897658824.meta, suffix=.meta, logDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824, archiveDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/oldWALs, maxLogs=32 2024-12-11T06:14:22,968 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824/4518703b2c1d%2C35425%2C1733897658824.meta.1733897662949.meta, exclude list is [], retry=0 2024-12-11T06:14:22,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33105,DS-73cb111b-9445-42c1-a84f-533c2b23174b,DISK] 2024-12-11T06:14:22,976 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/WALs/4518703b2c1d,35425,1733897658824/4518703b2c1d%2C35425%2C1733897658824.meta.1733897662949.meta 2024-12-11T06:14:22,977 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-12-11T06:14:22,977 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:14:22,979 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T06:14:22,982 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T06:14:22,986 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T06:14:22,991 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T06:14:22,992 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:22,992 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T06:14:22,993 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T06:14:22,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T06:14:22,998 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T06:14:22,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:23,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:23,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T06:14:23,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T06:14:23,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:23,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:23,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T06:14:23,005 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T06:14:23,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:23,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:23,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T06:14:23,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T06:14:23,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:23,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T06:14:23,010 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T06:14:23,012 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740 2024-12-11T06:14:23,015 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740 2024-12-11T06:14:23,017 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T06:14:23,018 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T06:14:23,019 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:14:23,022 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T06:14:23,024 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75059724, jitterRate=0.11847704648971558}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:14:23,024 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T06:14:23,026 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733897662993Writing region info on filesystem at 1733897662993Initializing all the Stores at 1733897662996 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662996Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662996Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897662996Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733897662996Cleaning up temporary data from old regions at 1733897663018 (+22 ms)Running coprocessor post-open hooks at 1733897663024 (+6 ms)Region opened successfully at 1733897663026 (+2 ms) 2024-12-11T06:14:23,034 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733897662918 2024-12-11T06:14:23,048 DEBUG [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T06:14:23,050 INFO [RS_OPEN_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T06:14:23,051 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:23,054 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4518703b2c1d,35425,1733897658824, state=OPEN 2024-12-11T06:14:23,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T06:14:23,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T06:14:23,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T06:14:23,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T06:14:23,139 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:23,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T06:14:23,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4518703b2c1d,35425,1733897658824 in 389 msec 2024-12-11T06:14:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T06:14:23,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 732 msec 2024-12-11T06:14:23,175 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T06:14:23,176 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T06:14:23,209 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:23,211 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:23,237 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:23,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:23,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8780 sec 2024-12-11T06:14:23,281 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733897663280, completionTime=-1 2024-12-11T06:14:23,284 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-11T06:14:23,285 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T06:14:23,317 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-11T06:14:23,317 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733897723317 2024-12-11T06:14:23,317 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733897783317 2024-12-11T06:14:23,317 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-11T06:14:23,320 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,321 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,321 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,322 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4518703b2c1d:41397, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,323 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,323 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,329 DEBUG [master/4518703b2c1d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T06:14:23,353 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.082sec 2024-12-11T06:14:23,354 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T06:14:23,356 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T06:14:23,357 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T06:14:23,358 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T06:14:23,358 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T06:14:23,359 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T06:14:23,359 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T06:14:23,369 DEBUG [master/4518703b2c1d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T06:14:23,371 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T06:14:23,371 INFO [master/4518703b2c1d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4518703b2c1d,41397,1733897657727-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T06:14:23,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5932f4ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:23,396 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T06:14:23,397 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T06:14:23,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:23,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:23,411 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4889feab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:23,421 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:23,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffd8077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:23,425 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:23,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:23,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:23,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:23,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4518703b2c1d,41397,1733897657727 2024-12-11T06:14:23,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=731, ProcessCount=11, AvailableMemoryMB=684 2024-12-11T06:14:23,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:14:23,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:14:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-12-11T06:14:23,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:14:23,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:14:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:14:23,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:14:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:23,589 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:14:23,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-11T06:14:23,596 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:23,600 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:14:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T06:14:23,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741835_1011 (size=1061) 2024-12-11T06:14:23,634 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:14:23,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741836_1012 (size=53) 2024-12-11T06:14:23,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:23,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 54b3d700051ccb5e9981f4a99fd0aae5, disabling compactions & flushes 2024-12-11T06:14:23,651 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:23,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:23,652 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. after waiting 0 ms 2024-12-11T06:14:23,652 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:23,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:23,652 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54b3d700051ccb5e9981f4a99fd0aae5: Waiting for close lock at 1733897663651Disabling compacts and flushes for region at 1733897663651Disabling writes for close at 1733897663652 (+1 ms)Writing region close event to WAL at 1733897663652Closed at 1733897663652 2024-12-11T06:14:23,655 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:14:23,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897663655"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897663655"}]},"ts":"1733897663655"} 2024-12-11T06:14:23,667 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:14:23,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:14:23,674 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897663670"}]},"ts":"1733897663670"} 2024-12-11T06:14:23,681 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:14:23,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, ASSIGN}] 2024-12-11T06:14:23,687 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, ASSIGN 2024-12-11T06:14:23,689 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:14:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T06:14:23,840 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=54b3d700051ccb5e9981f4a99fd0aae5, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:23,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, ASSIGN because future has completed 2024-12-11T06:14:23,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T06:14:24,006 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,007 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:14:24,007 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,007 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:24,007 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,008 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,014 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,019 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:24,020 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b3d700051ccb5e9981f4a99fd0aae5 columnFamilyName A 2024-12-11T06:14:24,020 DEBUG [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:24,022 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(327): Store=54b3d700051ccb5e9981f4a99fd0aae5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:24,023 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,025 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:24,025 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b3d700051ccb5e9981f4a99fd0aae5 columnFamilyName B 2024-12-11T06:14:24,025 DEBUG [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:24,026 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(327): Store=54b3d700051ccb5e9981f4a99fd0aae5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:24,026 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,030 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:24,031 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54b3d700051ccb5e9981f4a99fd0aae5 columnFamilyName C 2024-12-11T06:14:24,031 DEBUG [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:24,038 INFO [StoreOpener-54b3d700051ccb5e9981f4a99fd0aae5-1 {}] regionserver.HStore(327): Store=54b3d700051ccb5e9981f4a99fd0aae5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:24,039 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,039 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,042 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,042 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,045 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,045 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,046 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:14:24,050 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,066 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:14:24,067 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 54b3d700051ccb5e9981f4a99fd0aae5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68750978, jitterRate=0.02446940541267395}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:14:24,068 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,069 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 54b3d700051ccb5e9981f4a99fd0aae5: Running coprocessor pre-open hook at 1733897664008Writing region info on filesystem at 1733897664008Initializing all the Stores at 1733897664014 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897664014Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897664014Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897664014Cleaning up temporary data from old regions at 1733897664045 (+31 ms)Running coprocessor post-open hooks at 1733897664068 (+23 ms)Region opened successfully at 1733897664069 (+1 ms) 2024-12-11T06:14:24,072 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., pid=6, masterSystemTime=1733897664000 2024-12-11T06:14:24,076 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,076 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,078 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=54b3d700051ccb5e9981f4a99fd0aae5, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:14:24,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T06:14:24,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 in 239 msec 2024-12-11T06:14:24,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T06:14:24,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, ASSIGN in 407 msec 2024-12-11T06:14:24,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:14:24,098 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897664098"}]},"ts":"1733897664098"} 2024-12-11T06:14:24,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:14:24,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:14:24,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 530 msec 2024-12-11T06:14:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T06:14:24,240 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:24,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@737445c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a12729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,263 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,271 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:14:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@737a82e7 2024-12-11T06:14:24,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:14:24,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:14:24,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,304 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,314 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,320 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,325 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0722e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,337 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f8a4b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d3d4fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b28440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,362 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f79f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e54299b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ae3f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c865cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:24,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:24,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:24,395 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:24,408 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,410 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,412 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,414 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,415 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,415 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,418 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,427 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,424 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,445 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:24,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-11T06:14:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:24,479 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:24,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:24,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:24,492 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:14:24,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:24,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:14:24,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:24,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:24,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:24,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:24,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:24,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:24,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897724744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897724737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897724757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897724759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897724761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:24,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/dfe893ff268647ecb9e5f3c6bac23e17 is 50, key is test_row_0/A:col10/1733897664594/Put/seqid=0 2024-12-11T06:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741837_1013 (size=12001) 2024-12-11T06:14:24,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/dfe893ff268647ecb9e5f3c6bac23e17 2024-12-11T06:14:24,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:24,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:24,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897724932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897724933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897724933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897724934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:24,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897724943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/6a1fee743fe841509a46bf9b182a0582 is 50, key is test_row_0/B:col10/1733897664594/Put/seqid=0 2024-12-11T06:14:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:25,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897725160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897725161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897725162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897725179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897725178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741838_1014 (size=12001) 2024-12-11T06:14:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:25,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897725476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897725482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897725485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897725504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897725521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:25,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/6a1fee743fe841509a46bf9b182a0582 2024-12-11T06:14:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:25,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/7f4580c32bc74bd3b363a8a3694a4f66 is 50, key is test_row_0/C:col10/1733897664594/Put/seqid=0 2024-12-11T06:14:25,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741839_1015 (size=12001) 2024-12-11T06:14:25,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/7f4580c32bc74bd3b363a8a3694a4f66 2024-12-11T06:14:25,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/dfe893ff268647ecb9e5f3c6bac23e17 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17 2024-12-11T06:14:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:25,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T06:14:25,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/6a1fee743fe841509a46bf9b182a0582 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582 2024-12-11T06:14:25,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T06:14:25,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/7f4580c32bc74bd3b363a8a3694a4f66 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66 2024-12-11T06:14:25,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T06:14:25,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 54b3d700051ccb5e9981f4a99fd0aae5 in 1222ms, sequenceid=15, compaction requested=false 2024-12-11T06:14:25,834 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-11T06:14:25,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:25,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T06:14:25,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:25,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:25,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5af0f39fc6044b2f83980a28f9df339d is 50, key is test_row_0/A:col10/1733897664743/Put/seqid=0 2024-12-11T06:14:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741840_1016 (size=12001) 2024-12-11T06:14:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897726104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897726107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897726126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897726133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897726137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897726241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897726251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897726251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897726253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897726271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,433 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5af0f39fc6044b2f83980a28f9df339d 2024-12-11T06:14:26,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897726463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897726477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897726478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897726478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897726493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f8ef82ac95594a98ad1b26092fc5884f is 50, key is test_row_0/B:col10/1733897664743/Put/seqid=0 2024-12-11T06:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741841_1017 (size=12001) 2024-12-11T06:14:26,580 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f8ef82ac95594a98ad1b26092fc5884f 2024-12-11T06:14:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2341fd1ac6854a269836ffccb9bd8c13 is 50, key is test_row_0/C:col10/1733897664743/Put/seqid=0 2024-12-11T06:14:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:26,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741842_1018 (size=12001) 2024-12-11T06:14:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2341fd1ac6854a269836ffccb9bd8c13 2024-12-11T06:14:26,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5af0f39fc6044b2f83980a28f9df339d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d 2024-12-11T06:14:26,723 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d, entries=150, sequenceid=34, filesize=11.7 K 2024-12-11T06:14:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f8ef82ac95594a98ad1b26092fc5884f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f 2024-12-11T06:14:26,745 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f, entries=150, sequenceid=34, filesize=11.7 K 2024-12-11T06:14:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2341fd1ac6854a269836ffccb9bd8c13 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13 2024-12-11T06:14:26,783 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13, entries=150, sequenceid=34, filesize=11.7 K 2024-12-11T06:14:26,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897726780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,799 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 54b3d700051ccb5e9981f4a99fd0aae5 in 878ms, sequenceid=34, compaction requested=false 2024-12-11T06:14:26,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-11T06:14:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-11T06:14:26,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T06:14:26,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-12-11T06:14:26,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3600 sec 2024-12-11T06:14:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:26,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:26,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:26,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5f0c0814aef740b9bd9ecb685c394ad7 is 50, key is test_row_0/A:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897726907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897726910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897726911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897726922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741843_1019 (size=19021) 2024-12-11T06:14:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897727029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897727029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897727039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897727040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897727240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897727240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897727259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897727259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897727289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5f0c0814aef740b9bd9ecb685c394ad7 2024-12-11T06:14:27,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 is 50, key is test_row_0/B:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:27,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741844_1020 (size=12001) 2024-12-11T06:14:27,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 2024-12-11T06:14:27,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/74d0d9b5fd8f4a59844460e95fffc66f is 50, key is test_row_0/C:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897727549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741845_1021 (size=12001) 2024-12-11T06:14:27,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/74d0d9b5fd8f4a59844460e95fffc66f 2024-12-11T06:14:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897727559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/5f0c0814aef740b9bd9ecb685c394ad7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7 2024-12-11T06:14:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897727579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7, entries=300, sequenceid=51, filesize=18.6 K 2024-12-11T06:14:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897727579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:27,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 2024-12-11T06:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:27,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:14:27,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/74d0d9b5fd8f4a59844460e95fffc66f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f 2024-12-11T06:14:27,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:14:27,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 54b3d700051ccb5e9981f4a99fd0aae5 in 825ms, sequenceid=51, compaction requested=true 2024-12-11T06:14:27,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:27,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:27,682 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:27,688 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:27,682 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:27,690 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:27,690 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:27,691 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.2 K 2024-12-11T06:14:27,697 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a1fee743fe841509a46bf9b182a0582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897664517 2024-12-11T06:14:27,698 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ef82ac95594a98ad1b26092fc5884f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733897664738 2024-12-11T06:14:27,700 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting a2acbe6bdeba4a1a9dc5c22fcdd2eae6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666117 2024-12-11T06:14:27,705 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:27,705 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:27,705 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:27,705 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=42.0 K 2024-12-11T06:14:27,710 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe893ff268647ecb9e5f3c6bac23e17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897664517 2024-12-11T06:14:27,715 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af0f39fc6044b2f83980a28f9df339d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733897664738 2024-12-11T06:14:27,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T06:14:27,719 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0c0814aef740b9bd9ecb685c394ad7, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666090 2024-12-11T06:14:27,767 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#9 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:27,768 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152 is 50, key is test_row_0/B:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:27,820 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741846_1022 (size=12104) 2024-12-11T06:14:27,822 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/21c94f8f1d9e419897f2c9c7c043f5fb is 50, key is test_row_0/A:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:27,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741847_1023 (size=12104) 2024-12-11T06:14:27,918 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/21c94f8f1d9e419897f2c9c7c043f5fb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/21c94f8f1d9e419897f2c9c7c043f5fb 2024-12-11T06:14:27,958 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 21c94f8f1d9e419897f2c9c7c043f5fb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:27,958 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:27,971 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897667675; duration=0sec 2024-12-11T06:14:27,971 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:27,977 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:27,977 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:27,984 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:27,985 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:27,985 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:27,985 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.2 K 2024-12-11T06:14:27,990 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4580c32bc74bd3b363a8a3694a4f66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897664517 2024-12-11T06:14:27,991 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2341fd1ac6854a269836ffccb9bd8c13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733897664738 2024-12-11T06:14:27,992 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74d0d9b5fd8f4a59844460e95fffc66f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666117 2024-12-11T06:14:28,063 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#11 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:28,064 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e04b472f0a91424ba5d4fc2bb47581fa is 50, key is test_row_0/C:col10/1733897666117/Put/seqid=0 2024-12-11T06:14:28,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:28,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6c66c66ea63d49618c6ad2865c12a06d is 50, key is test_row_1/A:col10/1733897668061/Put/seqid=0 2024-12-11T06:14:28,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741848_1024 (size=12104) 2024-12-11T06:14:28,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741849_1025 (size=11997) 2024-12-11T06:14:28,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6c66c66ea63d49618c6ad2865c12a06d 2024-12-11T06:14:28,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c00d39e3e7dd4fdba93988ae8bcd46f7 is 50, key is test_row_1/B:col10/1733897668061/Put/seqid=0 2024-12-11T06:14:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897728151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897728157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897728204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897728213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741850_1026 (size=9657) 2024-12-11T06:14:28,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c00d39e3e7dd4fdba93988ae8bcd46f7 2024-12-11T06:14:28,268 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152 2024-12-11T06:14:28,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/523c26ea94964402a4ce7397c03e7e30 is 50, key is test_row_1/C:col10/1733897668061/Put/seqid=0 2024-12-11T06:14:28,309 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 7b4ccb9e0e7c4ac9b5f5bc1da1edb152(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:28,309 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:28,309 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897667679; duration=0sec 2024-12-11T06:14:28,309 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:28,310 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897728309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,315 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 54b3d700051ccb5e9981f4a99fd0aae5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897728321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741851_1027 (size=9657) 2024-12-11T06:14:28,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/523c26ea94964402a4ce7397c03e7e30 2024-12-11T06:14:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897728341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897728352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6c66c66ea63d49618c6ad2865c12a06d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d 2024-12-11T06:14:28,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897728352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d, entries=150, sequenceid=68, filesize=11.7 K 2024-12-11T06:14:28,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c00d39e3e7dd4fdba93988ae8bcd46f7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7 2024-12-11T06:14:28,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7, entries=100, sequenceid=68, filesize=9.4 K 2024-12-11T06:14:28,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/523c26ea94964402a4ce7397c03e7e30 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30 2024-12-11T06:14:28,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30, entries=100, sequenceid=68, filesize=9.4 K 2024-12-11T06:14:28,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 54b3d700051ccb5e9981f4a99fd0aae5 in 369ms, sequenceid=68, compaction requested=false 2024-12-11T06:14:28,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:28,504 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:14:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:28,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:28,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3119be60f4d6478ab31ff89a62828f5f is 50, key is test_row_0/A:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:28,566 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e04b472f0a91424ba5d4fc2bb47581fa as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e04b472f0a91424ba5d4fc2bb47581fa 2024-12-11T06:14:28,583 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into e04b472f0a91424ba5d4fc2bb47581fa(size=11.8 K), total size for store is 21.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:28,583 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:28,583 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897667679; duration=0sec 2024-12-11T06:14:28,584 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:28,584 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741852_1028 (size=14341) 2024-12-11T06:14:28,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897728615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897728620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897728620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897728623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,658 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T06:14:28,659 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T06:14:28,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T06:14:28,664 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T06:14:28,665 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T06:14:28,665 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T06:14:28,667 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-11T06:14:28,667 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-11T06:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T06:14:28,669 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:28,670 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-11T06:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:28,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:28,684 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:28,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897728731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897728743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897728743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897728743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:28,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:28,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:28,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897728939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897728959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897728968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897728959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:28,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:29,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3119be60f4d6478ab31ff89a62828f5f 2024-12-11T06:14:29,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/b88155bc852d4947b13d558f629c4d01 is 50, key is test_row_0/B:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:29,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741853_1029 (size=12001) 2024-12-11T06:14:29,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/b88155bc852d4947b13d558f629c4d01 2024-12-11T06:14:29,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/be466ba4bb174afda24161ebb7a0b030 is 50, key is test_row_0/C:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:29,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:29,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741854_1030 (size=12001) 2024-12-11T06:14:29,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897729249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897729282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897729282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897729289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:29,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/be466ba4bb174afda24161ebb7a0b030 2024-12-11T06:14:29,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3119be60f4d6478ab31ff89a62828f5f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f 2024-12-11T06:14:29,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:29,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f, entries=200, sequenceid=87, filesize=14.0 K 2024-12-11T06:14:29,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/b88155bc852d4947b13d558f629c4d01 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01 2024-12-11T06:14:29,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01, entries=150, sequenceid=87, filesize=11.7 K 2024-12-11T06:14:29,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/be466ba4bb174afda24161ebb7a0b030 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030 2024-12-11T06:14:29,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030, entries=150, sequenceid=87, filesize=11.7 K 2024-12-11T06:14:29,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 54b3d700051ccb5e9981f4a99fd0aae5 in 1212ms, sequenceid=87, compaction requested=true 2024-12-11T06:14:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:29,750 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:29,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:29,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:29,751 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:29,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:29,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:29,757 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:29,758 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:29,758 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,758 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=33.0 K 2024-12-11T06:14:29,758 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:29,759 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:29,759 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,759 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/21c94f8f1d9e419897f2c9c7c043f5fb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=37.5 K 2024-12-11T06:14:29,759 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4ccb9e0e7c4ac9b5f5bc1da1edb152, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666117 2024-12-11T06:14:29,760 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c00d39e3e7dd4fdba93988ae8bcd46f7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733897666916 2024-12-11T06:14:29,760 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c94f8f1d9e419897f2c9c7c043f5fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666117 2024-12-11T06:14:29,763 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b88155bc852d4947b13d558f629c4d01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:29,764 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c66c66ea63d49618c6ad2865c12a06d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733897666907 2024-12-11T06:14:29,767 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3119be60f4d6478ab31ff89a62828f5f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:29,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T06:14:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,794 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T06:14:29,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:29,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:29,815 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:29,816 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/2f223b9ddd9f4f169185ac9b3e39175e is 50, key is test_row_0/B:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:29,820 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:29,821 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0dfeb9225da44b9192c9c4fe2aa36f70 is 50, key is test_row_0/A:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:29,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/fa08b84d13a348ae9d135518bd8a5280 is 50, key is test_row_0/A:col10/1733897668616/Put/seqid=0 2024-12-11T06:14:29,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741856_1032 (size=12207) 2024-12-11T06:14:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897729865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897729865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897729871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741857_1033 (size=14341) 2024-12-11T06:14:29,894 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/fa08b84d13a348ae9d135518bd8a5280 2024-12-11T06:14:29,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897729879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741855_1031 (size=12207) 2024-12-11T06:14:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a717b59e15da4d81a6c013a17e216fb8 is 50, key is test_row_0/B:col10/1733897668616/Put/seqid=0 2024-12-11T06:14:29,941 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/2f223b9ddd9f4f169185ac9b3e39175e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/2f223b9ddd9f4f169185ac9b3e39175e 2024-12-11T06:14:29,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741858_1034 (size=12001) 2024-12-11T06:14:29,959 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a717b59e15da4d81a6c013a17e216fb8 2024-12-11T06:14:29,964 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 2f223b9ddd9f4f169185ac9b3e39175e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:29,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:29,965 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897669751; duration=0sec 2024-12-11T06:14:29,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:29,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:29,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:29,985 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:29,985 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:29,985 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:29,985 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e04b472f0a91424ba5d4fc2bb47581fa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=33.0 K 2024-12-11T06:14:29,986 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e04b472f0a91424ba5d4fc2bb47581fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897666117 2024-12-11T06:14:29,988 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 523c26ea94964402a4ce7397c03e7e30, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733897666916 2024-12-11T06:14:29,989 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting be466ba4bb174afda24161ebb7a0b030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2a48a7406fb54a89bae0bdb7e4f50c4f is 50, key is test_row_0/C:col10/1733897668616/Put/seqid=0 2024-12-11T06:14:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897729990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897729990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897729990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897730011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741859_1035 (size=12001) 2024-12-11T06:14:30,035 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:30,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2a48a7406fb54a89bae0bdb7e4f50c4f 2024-12-11T06:14:30,036 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3102fa6b38414cc6a5a388924da2ebb6 is 50, key is test_row_0/C:col10/1733897668531/Put/seqid=0 2024-12-11T06:14:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741860_1036 (size=12207) 2024-12-11T06:14:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/fa08b84d13a348ae9d135518bd8a5280 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280 2024-12-11T06:14:30,107 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3102fa6b38414cc6a5a388924da2ebb6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3102fa6b38414cc6a5a388924da2ebb6 2024-12-11T06:14:30,107 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280, entries=200, sequenceid=103, filesize=14.0 K 2024-12-11T06:14:30,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/a717b59e15da4d81a6c013a17e216fb8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8 2024-12-11T06:14:30,134 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8, entries=150, sequenceid=103, filesize=11.7 K 2024-12-11T06:14:30,134 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 3102fa6b38414cc6a5a388924da2ebb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:30,134 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:30,134 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897669751; duration=0sec 2024-12-11T06:14:30,134 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:30,134 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:30,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2a48a7406fb54a89bae0bdb7e4f50c4f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f 2024-12-11T06:14:30,151 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f, entries=150, sequenceid=103, filesize=11.7 K 2024-12-11T06:14:30,153 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 54b3d700051ccb5e9981f4a99fd0aae5 in 359ms, sequenceid=103, compaction requested=false 2024-12-11T06:14:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-11T06:14:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-11T06:14:30,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-11T06:14:30,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-12-11T06:14:30,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4990 sec 2024-12-11T06:14:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:30,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:14:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:30,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/36df3419ae9d4cac8ac0eb41a7400f3a is 50, key is test_row_0/A:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:30,291 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0dfeb9225da44b9192c9c4fe2aa36f70 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0dfeb9225da44b9192c9c4fe2aa36f70 2024-12-11T06:14:30,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897730281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897730284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897730284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897730291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741861_1037 (size=14341) 2024-12-11T06:14:30,312 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 0dfeb9225da44b9192c9c4fe2aa36f70(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:30,319 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:30,319 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897669750; duration=0sec 2024-12-11T06:14:30,319 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:30,319 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:30,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897730339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,347 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 54b3d700051ccb5e9981f4a99fd0aae5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:14:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897730405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897730620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897730620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897730627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897730627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/36df3419ae9d4cac8ac0eb41a7400f3a 2024-12-11T06:14:30,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e7238b66b416423580f3a23751c3ac1b is 50, key is test_row_0/B:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:30,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741862_1038 (size=12001) 2024-12-11T06:14:30,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e7238b66b416423580f3a23751c3ac1b 2024-12-11T06:14:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T06:14:30,829 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:30,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-11T06:14:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:30,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/8d6ec466db454b63b7ce98c114d5387b is 50, key is test_row_0/C:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:30,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:30,850 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:30,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:30,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741863_1039 (size=12001) 2024-12-11T06:14:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897730941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897730941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897730941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897730950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:31,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/8d6ec466db454b63b7ce98c114d5387b 2024-12-11T06:14:31,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/36df3419ae9d4cac8ac0eb41a7400f3a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a 2024-12-11T06:14:31,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a, entries=200, sequenceid=123, filesize=14.0 K 2024-12-11T06:14:31,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e7238b66b416423580f3a23751c3ac1b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b 2024-12-11T06:14:31,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b, entries=150, sequenceid=123, filesize=11.7 K 2024-12-11T06:14:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/8d6ec466db454b63b7ce98c114d5387b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b 2024-12-11T06:14:31,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b, entries=150, sequenceid=123, filesize=11.7 K 2024-12-11T06:14:31,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 54b3d700051ccb5e9981f4a99fd0aae5 in 1207ms, sequenceid=123, compaction requested=true 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:31,414 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:31,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:14:31,414 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:31,416 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:31,416 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:31,416 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,416 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:31,417 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/2f223b9ddd9f4f169185ac9b3e39175e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.4 K 2024-12-11T06:14:31,417 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:31,417 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,417 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0dfeb9225da44b9192c9c4fe2aa36f70, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=39.9 K 2024-12-11T06:14:31,418 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f223b9ddd9f4f169185ac9b3e39175e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:31,418 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dfeb9225da44b9192c9c4fe2aa36f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:31,420 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fa08b84d13a348ae9d135518bd8a5280, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733897668540 2024-12-11T06:14:31,420 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a717b59e15da4d81a6c013a17e216fb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733897668540 2024-12-11T06:14:31,423 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 36df3419ae9d4cac8ac0eb41a7400f3a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:31,423 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7238b66b416423580f3a23751c3ac1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:31,458 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:31,460 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0f7744fc70d44dfcae93fb2fe37150be is 50, key is test_row_0/A:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:31,463 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:31,464 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/df729464ee3e42dbb04308b60ec12124 is 50, key is test_row_0/B:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:31,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:14:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:31,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:31,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:31,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/4bb774c0646944e09540dcae35c26747 is 50, key is test_row_0/A:col10/1733897671461/Put/seqid=0 2024-12-11T06:14:31,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741865_1041 (size=12309) 2024-12-11T06:14:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741864_1040 (size=12309) 2024-12-11T06:14:31,536 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0f7744fc70d44dfcae93fb2fe37150be as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7744fc70d44dfcae93fb2fe37150be 2024-12-11T06:14:31,536 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/df729464ee3e42dbb04308b60ec12124 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/df729464ee3e42dbb04308b60ec12124 2024-12-11T06:14:31,560 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into df729464ee3e42dbb04308b60ec12124(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:31,560 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:31,560 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897671414; duration=0sec 2024-12-11T06:14:31,562 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:31,562 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:31,562 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:31,565 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 0f7744fc70d44dfcae93fb2fe37150be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:31,565 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:31,565 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897671414; duration=0sec 2024-12-11T06:14:31,565 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:31,565 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:31,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,567 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897731535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,568 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:31,568 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,568 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3102fa6b38414cc6a5a388924da2ebb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.4 K 2024-12-11T06:14:31,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897731539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,569 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3102fa6b38414cc6a5a388924da2ebb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897668140 2024-12-11T06:14:31,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741866_1042 (size=16931) 2024-12-11T06:14:31,570 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a48a7406fb54a89bae0bdb7e4f50c4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733897668540 2024-12-11T06:14:31,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,571 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6ec466db454b63b7ce98c114d5387b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/4bb774c0646944e09540dcae35c26747 2024-12-11T06:14:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897731546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897731554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9a6e02a457544b11a7e5f1d2c8683744 is 50, key is test_row_0/B:col10/1733897671461/Put/seqid=0 2024-12-11T06:14:31,618 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:31,619 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/cbc422d32ab04eb289f5251891d23dc9 is 50, key is test_row_0/C:col10/1733897669861/Put/seqid=0 2024-12-11T06:14:31,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741867_1043 (size=12151) 2024-12-11T06:14:31,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9a6e02a457544b11a7e5f1d2c8683744 2024-12-11T06:14:31,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3fad3e12bc724f858707fa2fdce7ec5f is 50, key is test_row_0/C:col10/1733897671461/Put/seqid=0 2024-12-11T06:14:31,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741868_1044 (size=12309) 2024-12-11T06:14:31,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897731697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897731698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897731697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897731698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741869_1045 (size=12151) 2024-12-11T06:14:31,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897731913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897731919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897731919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897731919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:31,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:31,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,106 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/cbc422d32ab04eb289f5251891d23dc9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/cbc422d32ab04eb289f5251891d23dc9 2024-12-11T06:14:32,133 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into cbc422d32ab04eb289f5251891d23dc9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:32,133 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,133 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897671414; duration=0sec 2024-12-11T06:14:32,134 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:32,134 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:32,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3fad3e12bc724f858707fa2fdce7ec5f 2024-12-11T06:14:32,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/4bb774c0646944e09540dcae35c26747 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747 2024-12-11T06:14:32,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747, entries=250, sequenceid=140, filesize=16.5 K 2024-12-11T06:14:32,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9a6e02a457544b11a7e5f1d2c8683744 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744 2024-12-11T06:14:32,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744, entries=150, sequenceid=140, filesize=11.9 K 2024-12-11T06:14:32,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3fad3e12bc724f858707fa2fdce7ec5f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f 2024-12-11T06:14:32,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f, entries=150, sequenceid=140, filesize=11.9 K 2024-12-11T06:14:32,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 54b3d700051ccb5e9981f4a99fd0aae5 in 750ms, sequenceid=140, compaction requested=false 2024-12-11T06:14:32,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:32,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:14:32,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:32,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:32,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:32,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/11a93b3b76b74176ad5af7644215f525 is 50, key is test_row_0/A:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741870_1046 (size=14541) 2024-12-11T06:14:32,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/11a93b3b76b74176ad5af7644215f525 2024-12-11T06:14:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897732261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897732263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897732266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897732268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/5f5f224fb15549fea33943675b99697f is 50, key is test_row_0/B:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:32,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:32,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:32,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741871_1047 (size=12151) 2024-12-11T06:14:32,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/5f5f224fb15549fea33943675b99697f 2024-12-11T06:14:32,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/46b5b6c72c81404081d5558b804e8e31 is 50, key is test_row_0/C:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741872_1048 (size=12151) 2024-12-11T06:14:32,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/46b5b6c72c81404081d5558b804e8e31 2024-12-11T06:14:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897732378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897732379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897732379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897732379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/11a93b3b76b74176ad5af7644215f525 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525 2024-12-11T06:14:32,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525, entries=200, sequenceid=160, filesize=14.2 K 2024-12-11T06:14:32,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/5f5f224fb15549fea33943675b99697f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f 2024-12-11T06:14:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f, entries=150, sequenceid=160, filesize=11.9 K 2024-12-11T06:14:32,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/46b5b6c72c81404081d5558b804e8e31 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31 2024-12-11T06:14:32,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31, entries=150, sequenceid=160, filesize=11.9 K 2024-12-11T06:14:32,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 54b3d700051ccb5e9981f4a99fd0aae5 in 192ms, sequenceid=160, compaction requested=true 2024-12-11T06:14:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:32,425 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:32,425 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:32,429 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43781 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:32,429 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:32,429 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,429 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7744fc70d44dfcae93fb2fe37150be, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=42.8 K 2024-12-11T06:14:32,430 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7744fc70d44dfcae93fb2fe37150be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:32,431 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:32,431 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:32,431 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,431 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/df729464ee3e42dbb04308b60ec12124, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.8 K 2024-12-11T06:14:32,432 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb774c0646944e09540dcae35c26747, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733897670259 2024-12-11T06:14:32,432 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting df729464ee3e42dbb04308b60ec12124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:32,432 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6e02a457544b11a7e5f1d2c8683744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733897670259 2024-12-11T06:14:32,432 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a93b3b76b74176ad5af7644215f525, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:32,433 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5f224fb15549fea33943675b99697f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:32,453 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:32,454 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/1767c2f37c55430589bb8e50d136efc8 is 50, key is test_row_0/B:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T06:14:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,460 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:32,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:32,462 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:32,463 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/af18ccfdaead40c3a46555a44c7c659d is 50, key is test_row_0/A:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/b20ccb90ff4541d490762c5df649f6d4 is 50, key is test_row_0/A:col10/1733897672263/Put/seqid=0 2024-12-11T06:14:32,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741873_1049 (size=12561) 2024-12-11T06:14:32,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741874_1050 (size=12561) 2024-12-11T06:14:32,499 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/af18ccfdaead40c3a46555a44c7c659d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/af18ccfdaead40c3a46555a44c7c659d 2024-12-11T06:14:32,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741875_1051 (size=12151) 2024-12-11T06:14:32,506 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/b20ccb90ff4541d490762c5df649f6d4 2024-12-11T06:14:32,518 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into af18ccfdaead40c3a46555a44c7c659d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:32,519 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,519 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897672425; duration=0sec 2024-12-11T06:14:32,519 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:32,519 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:32,519 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:32,522 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:32,522 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:32,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f140dbb6d26e48fb9e2fd40f2abb0c94 is 50, key is test_row_0/B:col10/1733897672263/Put/seqid=0 2024-12-11T06:14:32,522 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:32,522 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/cbc422d32ab04eb289f5251891d23dc9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=35.8 K 2024-12-11T06:14:32,523 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc422d32ab04eb289f5251891d23dc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733897669861 2024-12-11T06:14:32,525 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fad3e12bc724f858707fa2fdce7ec5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733897670259 2024-12-11T06:14:32,525 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46b5b6c72c81404081d5558b804e8e31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:32,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741876_1052 (size=12151) 2024-12-11T06:14:32,545 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f140dbb6d26e48fb9e2fd40f2abb0c94 2024-12-11T06:14:32,547 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:32,547 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/70f0e8d28fa546c8b8075a29ce378851 is 50, key is test_row_0/C:col10/1733897672230/Put/seqid=0 2024-12-11T06:14:32,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/d71e78f3c3974f9a9165e8eb43779718 is 50, key is test_row_0/C:col10/1733897672263/Put/seqid=0 2024-12-11T06:14:32,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741877_1053 (size=12561) 2024-12-11T06:14:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:32,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741878_1054 (size=12151) 2024-12-11T06:14:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897732614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897732618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897732620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897732621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897732729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897732729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897732729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897732729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,893 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/1767c2f37c55430589bb8e50d136efc8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1767c2f37c55430589bb8e50d136efc8 2024-12-11T06:14:32,902 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 1767c2f37c55430589bb8e50d136efc8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:32,902 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,902 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897672425; duration=0sec 2024-12-11T06:14:32,902 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:32,902 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897732938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897732938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897732938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897732939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:32,985 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/70f0e8d28fa546c8b8075a29ce378851 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/70f0e8d28fa546c8b8075a29ce378851 2024-12-11T06:14:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:32,995 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 70f0e8d28fa546c8b8075a29ce378851(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:32,995 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:32,995 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897672426; duration=0sec 2024-12-11T06:14:32,995 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:32,995 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:32,997 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/d71e78f3c3974f9a9165e8eb43779718 2024-12-11T06:14:33,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/b20ccb90ff4541d490762c5df649f6d4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4 2024-12-11T06:14:33,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4, entries=150, sequenceid=175, filesize=11.9 K 2024-12-11T06:14:33,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/f140dbb6d26e48fb9e2fd40f2abb0c94 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94 2024-12-11T06:14:33,037 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94, entries=150, sequenceid=175, filesize=11.9 K 2024-12-11T06:14:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/d71e78f3c3974f9a9165e8eb43779718 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718 2024-12-11T06:14:33,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718, entries=150, sequenceid=175, filesize=11.9 K 2024-12-11T06:14:33,052 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 54b3d700051ccb5e9981f4a99fd0aae5 in 592ms, sequenceid=175, compaction requested=false 2024-12-11T06:14:33,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:33,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:33,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-11T06:14:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-11T06:14:33,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-11T06:14:33,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2050 sec 2024-12-11T06:14:33,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.2240 sec 2024-12-11T06:14:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:33,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:33,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/441e52c29eaf4474ad1ad4ef7ebdee27 is 50, key is test_row_0/A:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897733276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897733278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897733278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897733279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741879_1055 (size=16931) 2024-12-11T06:14:33,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/441e52c29eaf4474ad1ad4ef7ebdee27 2024-12-11T06:14:33,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72b8b9926b37443e966def0c080956e4 is 50, key is test_row_0/B:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741880_1056 (size=12151) 2024-12-11T06:14:33,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72b8b9926b37443e966def0c080956e4 2024-12-11T06:14:33,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c3ea842473714c959368feac6d1375e4 is 50, key is test_row_0/C:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741881_1057 (size=12151) 2024-12-11T06:14:33,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c3ea842473714c959368feac6d1375e4 2024-12-11T06:14:33,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/441e52c29eaf4474ad1ad4ef7ebdee27 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27 2024-12-11T06:14:33,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897733390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897733391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897733391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897733391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27, entries=250, sequenceid=197, filesize=16.5 K 2024-12-11T06:14:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72b8b9926b37443e966def0c080956e4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4 2024-12-11T06:14:33,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T06:14:33,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c3ea842473714c959368feac6d1375e4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4 2024-12-11T06:14:33,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T06:14:33,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 54b3d700051ccb5e9981f4a99fd0aae5 in 178ms, sequenceid=197, compaction requested=true 2024-12-11T06:14:33,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:33,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:33,429 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:33,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:33,430 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:33,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:33,435 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:33,435 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:33,435 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:33,435 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1767c2f37c55430589bb8e50d136efc8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.0 K 2024-12-11T06:14:33,436 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:33,436 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:33,436 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:33,436 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/af18ccfdaead40c3a46555a44c7c659d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=40.7 K 2024-12-11T06:14:33,436 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1767c2f37c55430589bb8e50d136efc8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:33,437 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting af18ccfdaead40c3a46555a44c7c659d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:33,438 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f140dbb6d26e48fb9e2fd40f2abb0c94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733897672260 2024-12-11T06:14:33,438 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20ccb90ff4541d490762c5df649f6d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733897672260 2024-12-11T06:14:33,439 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b8b9926b37443e966def0c080956e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:33,440 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441e52c29eaf4474ad1ad4ef7ebdee27, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:33,475 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:33,477 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c5ebe7147c014e228677cad9f0e06faa is 50, key is test_row_0/B:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,486 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:33,486 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/964e542c6e3748c4bf16d7fb0e977b9d is 50, key is test_row_0/A:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741882_1058 (size=12663) 2024-12-11T06:14:33,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741883_1059 (size=12663) 2024-12-11T06:14:33,524 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c5ebe7147c014e228677cad9f0e06faa as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c5ebe7147c014e228677cad9f0e06faa 2024-12-11T06:14:33,536 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into c5ebe7147c014e228677cad9f0e06faa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:33,536 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:33,536 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897673429; duration=0sec 2024-12-11T06:14:33,536 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:33,536 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:33,536 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:33,539 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:33,539 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:33,539 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:33,540 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/70f0e8d28fa546c8b8075a29ce378851, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.0 K 2024-12-11T06:14:33,540 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 70f0e8d28fa546c8b8075a29ce378851, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897671525 2024-12-11T06:14:33,540 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d71e78f3c3974f9a9165e8eb43779718, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733897672260 2024-12-11T06:14:33,541 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ea842473714c959368feac6d1375e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:33,555 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:33,556 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/91de2bb8c90c455ab5533415ca987f71 is 50, key is test_row_0/C:col10/1733897673251/Put/seqid=0 2024-12-11T06:14:33,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741884_1060 (size=12663) 2024-12-11T06:14:33,597 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/91de2bb8c90c455ab5533415ca987f71 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/91de2bb8c90c455ab5533415ca987f71 2024-12-11T06:14:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:33,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:14:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:33,610 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 91de2bb8c90c455ab5533415ca987f71(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:33,610 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:33,610 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897673430; duration=0sec 2024-12-11T06:14:33,610 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:33,610 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:33,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/82a878eb952d40769f85a585a7785cb5 is 50, key is test_row_0/A:col10/1733897673602/Put/seqid=0 2024-12-11T06:14:33,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897733630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897733630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897733633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897733634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741885_1061 (size=14541) 2024-12-11T06:14:33,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/82a878eb952d40769f85a585a7785cb5 2024-12-11T06:14:33,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e032e7930a914e7c9cb39c96deb8853d is 50, key is test_row_0/B:col10/1733897673602/Put/seqid=0 2024-12-11T06:14:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741886_1062 (size=12151) 2024-12-11T06:14:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897733738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897733738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897733749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897733749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,921 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/964e542c6e3748c4bf16d7fb0e977b9d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/964e542c6e3748c4bf16d7fb0e977b9d 2024-12-11T06:14:33,928 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 964e542c6e3748c4bf16d7fb0e977b9d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:33,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:33,928 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897673429; duration=0sec 2024-12-11T06:14:33,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:33,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897733949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897733949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897733958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897733959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e032e7930a914e7c9cb39c96deb8853d 2024-12-11T06:14:34,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/391e4e644a03469dac5ae23466cbc43e is 50, key is test_row_0/C:col10/1733897673602/Put/seqid=0 2024-12-11T06:14:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741887_1063 (size=12151) 2024-12-11T06:14:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897734258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897734258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897734269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897734269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897734378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,380 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 54b3d700051ccb5e9981f4a99fd0aae5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:14:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/391e4e644a03469dac5ae23466cbc43e 2024-12-11T06:14:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/82a878eb952d40769f85a585a7785cb5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5 2024-12-11T06:14:34,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5, entries=200, sequenceid=214, filesize=14.2 K 2024-12-11T06:14:34,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e032e7930a914e7c9cb39c96deb8853d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d 2024-12-11T06:14:34,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d, entries=150, sequenceid=214, filesize=11.9 K 2024-12-11T06:14:34,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/391e4e644a03469dac5ae23466cbc43e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e 2024-12-11T06:14:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e, entries=150, sequenceid=214, filesize=11.9 K 2024-12-11T06:14:34,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 54b3d700051ccb5e9981f4a99fd0aae5 in 932ms, sequenceid=214, compaction requested=false 2024-12-11T06:14:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:34,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:14:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:34,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:34,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/afd379e83d3d4fc2a3f82d8649d5abff is 50, key is test_row_0/A:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:34,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741888_1064 (size=12151) 2024-12-11T06:14:34,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897734789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897734793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897734795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897734795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897734908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897734908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897734909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897734909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T06:14:34,999 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:34,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-11T06:14:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:35,002 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:35,004 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:35,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897735118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897735119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897735119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897735119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T06:14:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/afd379e83d3d4fc2a3f82d8649d5abff 2024-12-11T06:14:35,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/669990fa4a9943389bf3ced20954fd72 is 50, key is test_row_0/B:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741889_1065 (size=12151) 2024-12-11T06:14:35,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/669990fa4a9943389bf3ced20954fd72 2024-12-11T06:14:35,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/39b00a3c47ad4169b0df19771cfaf265 is 50, key is test_row_0/C:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:35,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741890_1066 (size=12151) 2024-12-11T06:14:35,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T06:14:35,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897735428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897735429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897735429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897735438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T06:14:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T06:14:35,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:35,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/39b00a3c47ad4169b0df19771cfaf265 2024-12-11T06:14:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:35,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/afd379e83d3d4fc2a3f82d8649d5abff as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff 2024-12-11T06:14:35,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff, entries=150, sequenceid=234, filesize=11.9 K 2024-12-11T06:14:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/669990fa4a9943389bf3ced20954fd72 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72 2024-12-11T06:14:35,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72, entries=150, sequenceid=234, filesize=11.9 K 2024-12-11T06:14:35,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/39b00a3c47ad4169b0df19771cfaf265 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265 2024-12-11T06:14:35,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265, entries=150, sequenceid=234, filesize=11.9 K 2024-12-11T06:14:35,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 54b3d700051ccb5e9981f4a99fd0aae5 in 882ms, sequenceid=234, compaction requested=true 2024-12-11T06:14:35,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:35,653 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:35,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:14:35,653 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:35,654 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:35,654 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:35,654 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:35,654 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:35,654 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,654 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,654 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c5ebe7147c014e228677cad9f0e06faa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.1 K 2024-12-11T06:14:35,654 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/964e542c6e3748c4bf16d7fb0e977b9d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=38.4 K 2024-12-11T06:14:35,655 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ebe7147c014e228677cad9f0e06faa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:35,655 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964e542c6e3748c4bf16d7fb0e977b9d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:35,655 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e032e7930a914e7c9cb39c96deb8853d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733897673276 2024-12-11T06:14:35,656 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 669990fa4a9943389bf3ced20954fd72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:35,656 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a878eb952d40769f85a585a7785cb5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733897673276 2024-12-11T06:14:35,656 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd379e83d3d4fc2a3f82d8649d5abff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:35,666 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:35,667 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/62160f255f47497ca1f51d9f8080d7c8 is 50, key is test_row_0/B:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:35,670 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:35,671 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/e7fa7486262741098c703cf78c184d30 is 50, key is test_row_0/A:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741891_1067 (size=12765) 2024-12-11T06:14:35,686 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/62160f255f47497ca1f51d9f8080d7c8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/62160f255f47497ca1f51d9f8080d7c8 2024-12-11T06:14:35,696 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 62160f255f47497ca1f51d9f8080d7c8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:35,696 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:35,696 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897675653; duration=0sec 2024-12-11T06:14:35,697 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:35,697 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:35,697 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:35,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741892_1068 (size=12765) 2024-12-11T06:14:35,699 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:35,699 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:35,700 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,700 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/91de2bb8c90c455ab5533415ca987f71, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.1 K 2024-12-11T06:14:35,701 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 91de2bb8c90c455ab5533415ca987f71, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897672612 2024-12-11T06:14:35,701 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 391e4e644a03469dac5ae23466cbc43e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733897673276 2024-12-11T06:14:35,701 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b00a3c47ad4169b0df19771cfaf265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:35,706 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/e7fa7486262741098c703cf78c184d30 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e7fa7486262741098c703cf78c184d30 2024-12-11T06:14:35,716 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into e7fa7486262741098c703cf78c184d30(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:35,716 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:35,717 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897675653; duration=0sec 2024-12-11T06:14:35,717 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:35,717 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:35,723 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:35,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e491fd9f59c54f0185f4ba66d6cbefc3 is 50, key is test_row_0/C:col10/1733897674769/Put/seqid=0 2024-12-11T06:14:35,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741893_1069 (size=12765) 2024-12-11T06:14:35,746 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e491fd9f59c54f0185f4ba66d6cbefc3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e491fd9f59c54f0185f4ba66d6cbefc3 2024-12-11T06:14:35,756 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into e491fd9f59c54f0185f4ba66d6cbefc3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:35,756 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:35,756 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897675653; duration=0sec 2024-12-11T06:14:35,757 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:35,757 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:35,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T06:14:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:35,771 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:35,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8f24e1ff7b3743a8926de93b783a3eb4 is 50, key is test_row_0/A:col10/1733897674780/Put/seqid=0 2024-12-11T06:14:35,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741894_1070 (size=12151) 2024-12-11T06:14:35,787 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8f24e1ff7b3743a8926de93b783a3eb4 2024-12-11T06:14:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e01bdc312add47f2b2866d7d367b5904 is 50, key is test_row_0/B:col10/1733897674780/Put/seqid=0 2024-12-11T06:14:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741895_1071 (size=12151) 2024-12-11T06:14:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:35,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897735984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897735982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897735986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897735989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897736099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897736099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897736099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897736099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:36,203 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e01bdc312add47f2b2866d7d367b5904 2024-12-11T06:14:36,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/773f4b6568514dfb91a307fcef4d7a03 is 50, key is test_row_0/C:col10/1733897674780/Put/seqid=0 2024-12-11T06:14:36,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741896_1072 (size=12151) 2024-12-11T06:14:36,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897736309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897736309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897736309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897736309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,616 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/773f4b6568514dfb91a307fcef4d7a03 2024-12-11T06:14:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897736619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897736619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897736619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8f24e1ff7b3743a8926de93b783a3eb4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4 2024-12-11T06:14:36,628 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4, entries=150, sequenceid=251, filesize=11.9 K 2024-12-11T06:14:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e01bdc312add47f2b2866d7d367b5904 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904 2024-12-11T06:14:36,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897736628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:36,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904, entries=150, sequenceid=251, filesize=11.9 K 2024-12-11T06:14:36,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/773f4b6568514dfb91a307fcef4d7a03 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03 2024-12-11T06:14:36,642 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03, entries=150, sequenceid=251, filesize=11.9 K 2024-12-11T06:14:36,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 54b3d700051ccb5e9981f4a99fd0aae5 in 872ms, sequenceid=251, compaction requested=false 2024-12-11T06:14:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-11T06:14:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-11T06:14:36,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T06:14:36,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-11T06:14:36,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6500 sec 2024-12-11T06:14:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:37,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:37,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T06:14:37,148 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:37,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897737165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897737166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897737175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897737175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-11T06:14:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:37,190 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:37,191 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:37,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:37,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8a68126706534070928afa01d4bf6d1b is 50, key is test_row_0/A:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:37,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741897_1073 (size=14741) 2024-12-11T06:14:37,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8a68126706534070928afa01d4bf6d1b 2024-12-11T06:14:37,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/82ae7d82eb7b442585d3053e4ac46908 is 50, key is test_row_0/B:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:37,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741898_1074 (size=12301) 2024-12-11T06:14:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897737278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897737278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897737289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897737289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:37,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897737498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897737498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:37,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/82ae7d82eb7b442585d3053e4ac46908 2024-12-11T06:14:37,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f5339688a806457ebdedf2a5281749cc is 50, key is test_row_0/C:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:37,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741899_1075 (size=12301) 2024-12-11T06:14:37,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897737799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897737799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897737808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897737808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:37,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:37,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:38,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f5339688a806457ebdedf2a5281749cc 2024-12-11T06:14:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/8a68126706534070928afa01d4bf6d1b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b 2024-12-11T06:14:38,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b, entries=200, sequenceid=271, filesize=14.4 K 2024-12-11T06:14:38,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/82ae7d82eb7b442585d3053e4ac46908 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908 2024-12-11T06:14:38,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908, entries=150, sequenceid=271, filesize=12.0 K 2024-12-11T06:14:38,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f5339688a806457ebdedf2a5281749cc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc 2024-12-11T06:14:38,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc, entries=150, sequenceid=271, filesize=12.0 K 2024-12-11T06:14:38,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 54b3d700051ccb5e9981f4a99fd0aae5 in 928ms, sequenceid=271, compaction requested=true 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:38,058 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:38,058 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:38,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:38,059 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:38,059 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:38,059 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:38,059 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:38,059 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:38,059 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:38,060 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/62160f255f47497ca1f51d9f8080d7c8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.3 K 2024-12-11T06:14:38,060 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e7fa7486262741098c703cf78c184d30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=38.7 K 2024-12-11T06:14:38,060 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7fa7486262741098c703cf78c184d30, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:38,060 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 62160f255f47497ca1f51d9f8080d7c8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:38,060 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f24e1ff7b3743a8926de93b783a3eb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733897674780 2024-12-11T06:14:38,060 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e01bdc312add47f2b2866d7d367b5904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733897674780 2024-12-11T06:14:38,061 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a68126706534070928afa01d4bf6d1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675980 2024-12-11T06:14:38,061 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ae7d82eb7b442585d3053e4ac46908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675984 2024-12-11T06:14:38,069 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:38,070 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9a23d609c92241c48205d6649e65b03d is 50, key is test_row_0/B:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:38,070 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:38,070 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/b54a09ae4f624464b18b198d3acf688e is 50, key is test_row_0/A:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741900_1076 (size=13017) 2024-12-11T06:14:38,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741901_1077 (size=13017) 2024-12-11T06:14:38,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:38,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3b36c65a6c404ee08a1f7608939cdeb8 is 50, key is test_row_0/A:col10/1733897677138/Put/seqid=0 2024-12-11T06:14:38,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741902_1078 (size=12301) 2024-12-11T06:14:38,136 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3b36c65a6c404ee08a1f7608939cdeb8 2024-12-11T06:14:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c1cc645cd82d4b97a5e85f85e55ad8e6 is 50, key is test_row_0/B:col10/1733897677138/Put/seqid=0 2024-12-11T06:14:38,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741903_1079 (size=12301) 2024-12-11T06:14:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897738332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897738336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897738337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897738337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897738439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897738448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897738448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897738448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,496 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/b54a09ae4f624464b18b198d3acf688e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b54a09ae4f624464b18b198d3acf688e 2024-12-11T06:14:38,501 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9a23d609c92241c48205d6649e65b03d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a23d609c92241c48205d6649e65b03d 2024-12-11T06:14:38,510 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into b54a09ae4f624464b18b198d3acf688e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:38,510 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:38,510 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897678058; duration=0sec 2024-12-11T06:14:38,512 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:38,512 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:38,512 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:38,513 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 9a23d609c92241c48205d6649e65b03d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:38,513 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:38,513 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897678058; duration=0sec 2024-12-11T06:14:38,513 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:38,513 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:38,514 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:38,514 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:38,514 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:38,514 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e491fd9f59c54f0185f4ba66d6cbefc3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.3 K 2024-12-11T06:14:38,515 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e491fd9f59c54f0185f4ba66d6cbefc3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733897673625 2024-12-11T06:14:38,515 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 773f4b6568514dfb91a307fcef4d7a03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733897674780 2024-12-11T06:14:38,516 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5339688a806457ebdedf2a5281749cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675984 2024-12-11T06:14:38,530 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:38,531 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/96ff66d3d78844da801b2da848509090 is 50, key is test_row_0/C:col10/1733897677129/Put/seqid=0 2024-12-11T06:14:38,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741904_1080 (size=13017) 2024-12-11T06:14:38,574 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c1cc645cd82d4b97a5e85f85e55ad8e6 2024-12-11T06:14:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2c20daec222a414aac9a03adc5ffd47b is 50, key is test_row_0/C:col10/1733897677138/Put/seqid=0 2024-12-11T06:14:38,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741905_1081 (size=12301) 2024-12-11T06:14:38,588 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2c20daec222a414aac9a03adc5ffd47b 2024-12-11T06:14:38,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3b36c65a6c404ee08a1f7608939cdeb8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8 2024-12-11T06:14:38,604 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8, entries=150, sequenceid=284, filesize=12.0 K 2024-12-11T06:14:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/c1cc645cd82d4b97a5e85f85e55ad8e6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6 2024-12-11T06:14:38,613 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6, entries=150, sequenceid=284, filesize=12.0 K 2024-12-11T06:14:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/2c20daec222a414aac9a03adc5ffd47b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b 2024-12-11T06:14:38,623 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b, entries=150, sequenceid=284, filesize=12.0 K 2024-12-11T06:14:38,624 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54b3d700051ccb5e9981f4a99fd0aae5 in 512ms, sequenceid=284, compaction requested=false 2024-12-11T06:14:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-11T06:14:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-11T06:14:38,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-11T06:14:38,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-12-11T06:14:38,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4800 sec 2024-12-11T06:14:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:38,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:14:38,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:38,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f93b32b058914616ad50bb1a959b26fc is 50, key is test_row_0/A:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:38,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741906_1082 (size=14741) 2024-12-11T06:14:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897738665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897738666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897738671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897738671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897738779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897738779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897738779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897738779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,947 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/96ff66d3d78844da801b2da848509090 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/96ff66d3d78844da801b2da848509090 2024-12-11T06:14:38,956 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 96ff66d3d78844da801b2da848509090(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:38,956 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:38,956 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897678058; duration=0sec 2024-12-11T06:14:38,956 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:38,956 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:38,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897738988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897738989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897738989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897738989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:38,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f93b32b058914616ad50bb1a959b26fc 2024-12-11T06:14:39,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9dcd40d6054d4793a60b3378cc1754e1 is 50, key is test_row_0/B:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:39,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741907_1083 (size=12301) 2024-12-11T06:14:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897739299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897739299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897739299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897739299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T06:14:39,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:39,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-11T06:14:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:39,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:39,353 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:39,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:39,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9dcd40d6054d4793a60b3378cc1754e1 2024-12-11T06:14:39,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f8f153ad9e374e7e92841bbf1596f805 is 50, key is test_row_0/C:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:39,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741908_1084 (size=12301) 2024-12-11T06:14:39,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T06:14:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T06:14:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:39,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:39,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T06:14:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:39,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897739819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897739819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897739819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897739819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:39,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f8f153ad9e374e7e92841bbf1596f805 2024-12-11T06:14:39,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f93b32b058914616ad50bb1a959b26fc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc 2024-12-11T06:14:39,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc, entries=200, sequenceid=306, filesize=14.4 K 2024-12-11T06:14:39,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/9dcd40d6054d4793a60b3378cc1754e1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1 2024-12-11T06:14:39,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1, entries=150, sequenceid=306, filesize=12.0 K 2024-12-11T06:14:39,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/f8f153ad9e374e7e92841bbf1596f805 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805 2024-12-11T06:14:39,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805, entries=150, sequenceid=306, filesize=12.0 K 2024-12-11T06:14:39,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54b3d700051ccb5e9981f4a99fd0aae5 in 1272ms, sequenceid=306, compaction requested=true 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:39,923 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:39,923 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:39,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:39,924 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:39,924 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:39,924 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,925 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a23d609c92241c48205d6649e65b03d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.7 K 2024-12-11T06:14:39,927 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:39,927 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a23d609c92241c48205d6649e65b03d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675984 2024-12-11T06:14:39,927 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:39,927 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,927 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b54a09ae4f624464b18b198d3acf688e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=39.1 K 2024-12-11T06:14:39,927 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c1cc645cd82d4b97a5e85f85e55ad8e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733897677138 2024-12-11T06:14:39,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54a09ae4f624464b18b198d3acf688e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675984 2024-12-11T06:14:39,928 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcd40d6054d4793a60b3378cc1754e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:39,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b36c65a6c404ee08a1f7608939cdeb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733897677138 2024-12-11T06:14:39,929 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93b32b058914616ad50bb1a959b26fc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:39,938 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:39,939 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/fc228c8c52ef4939925fff6ff9004b9c is 50, key is test_row_0/B:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:39,942 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:39,943 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/aef1f3308ffb4ba686ff4d0b0927a837 is 50, key is test_row_0/A:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741909_1085 (size=13119) 2024-12-11T06:14:39,954 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/fc228c8c52ef4939925fff6ff9004b9c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/fc228c8c52ef4939925fff6ff9004b9c 2024-12-11T06:14:39,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741910_1086 (size=13119) 2024-12-11T06:14:39,962 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into fc228c8c52ef4939925fff6ff9004b9c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:39,963 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:39,963 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897679923; duration=0sec 2024-12-11T06:14:39,963 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:39,963 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:39,963 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:39,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:39,965 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:39,965 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,965 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/96ff66d3d78844da801b2da848509090, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=36.7 K 2024-12-11T06:14:39,966 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ff66d3d78844da801b2da848509090, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733897675984 2024-12-11T06:14:39,967 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c20daec222a414aac9a03adc5ffd47b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733897677138 2024-12-11T06:14:39,967 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f153ad9e374e7e92841bbf1596f805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:39,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T06:14:39,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:39,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:39,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:39,971 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/aef1f3308ffb4ba686ff4d0b0927a837 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/aef1f3308ffb4ba686ff4d0b0927a837 2024-12-11T06:14:39,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/e33430614689482380a00896428d41a0 is 50, key is test_row_0/A:col10/1733897678664/Put/seqid=0 2024-12-11T06:14:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:39,980 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into aef1f3308ffb4ba686ff4d0b0927a837(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:39,980 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:39,980 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897679923; duration=0sec 2024-12-11T06:14:39,980 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:39,980 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:39,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741911_1087 (size=9857) 2024-12-11T06:14:39,990 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:39,990 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/7e3301da426c4ce993f987581b3199d3 is 50, key is test_row_0/C:col10/1733897678334/Put/seqid=0 2024-12-11T06:14:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741912_1088 (size=13119) 2024-12-11T06:14:40,387 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/e33430614689482380a00896428d41a0 2024-12-11T06:14:40,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72d820fa1bd343e3ad93bd8fcc6b27a4 is 50, key is test_row_0/B:col10/1733897678664/Put/seqid=0 2024-12-11T06:14:40,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741913_1089 (size=9857) 2024-12-11T06:14:40,407 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/7e3301da426c4ce993f987581b3199d3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7e3301da426c4ce993f987581b3199d3 2024-12-11T06:14:40,413 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 7e3301da426c4ce993f987581b3199d3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:40,413 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:40,413 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897679923; duration=0sec 2024-12-11T06:14:40,414 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:40,414 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:40,807 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72d820fa1bd343e3ad93bd8fcc6b27a4 2024-12-11T06:14:40,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e96f0b5c2439490f8db80baf719b8573 is 50, key is test_row_0/C:col10/1733897678664/Put/seqid=0 2024-12-11T06:14:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741914_1090 (size=9857) 2024-12-11T06:14:40,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:40,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897740851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897740851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897740853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897740855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897740959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897740968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897740968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897740968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897741168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897741178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897741179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897741179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,224 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e96f0b5c2439490f8db80baf719b8573 2024-12-11T06:14:41,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/e33430614689482380a00896428d41a0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0 2024-12-11T06:14:41,244 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0, entries=100, sequenceid=320, filesize=9.6 K 2024-12-11T06:14:41,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/72d820fa1bd343e3ad93bd8fcc6b27a4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4 2024-12-11T06:14:41,251 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4, entries=100, sequenceid=320, filesize=9.6 K 2024-12-11T06:14:41,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/e96f0b5c2439490f8db80baf719b8573 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573 2024-12-11T06:14:41,259 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573, entries=100, sequenceid=320, filesize=9.6 K 2024-12-11T06:14:41,260 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54b3d700051ccb5e9981f4a99fd0aae5 in 1290ms, sequenceid=320, compaction requested=false 2024-12-11T06:14:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-11T06:14:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-11T06:14:41,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-11T06:14:41,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9090 sec 2024-12-11T06:14:41,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9160 sec 2024-12-11T06:14:41,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897680839, max timestamp=1733897681479Num uniques 859; 2024-12-11T06:14:41,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897680839, max timestamp=1733897681479Num uniques 859; 2024-12-11T06:14:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:41,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-11T06:14:41,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:41,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897680839, max timestamp=1733897681479Num uniques 859; 2024-12-11T06:14:41,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:41,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:41,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T06:14:41,501 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:41,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-11T06:14:41,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/72db22ca37434560b94a1e5d7f86ec32 is 50, key is test_row_0/A:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:41,506 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:41,507 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:41,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741915_1091 (size=12301) 2024-12-11T06:14:41,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/72db22ca37434560b94a1e5d7f86ec32 2024-12-11T06:14:41,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 is 50, key is test_row_0/B:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:41,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741916_1092 (size=12301) 2024-12-11T06:14:41,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897681533, max timestamp=1733897681556Num uniques 859; 2024-12-11T06:14:41,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897681533, max timestamp=1733897681543Num uniques 859; 2024-12-11T06:14:41,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897681533, max timestamp=1733897681556Num uniques 859; 2024-12-11T06:14:41,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:41,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733897681592, max timestamp=1733897681614Num uniques 1018; 2024-12-11T06:14:41,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897681592, max timestamp=1733897681614Num uniques 1018; 2024-12-11T06:14:41,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-11T06:14:41,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733897681592, max timestamp=1733897681614Num uniques 1018; 2024-12-11T06:14:41,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-11T06:14:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733897681635, max timestamp=1733897681643Num uniques 1127; 2024-12-11T06:14:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-12-11T06:14:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733897681635, max timestamp=1733897681643Num uniques 1027; 2024-12-11T06:14:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12674, totalHeapSize=29048, min timestamp=1733897681635, max timestamp=1733897681644Num uniques 1127; 2024-12-11T06:14:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-11T06:14:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51604, new segment data size=12674, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-12-11T06:14:41,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897741673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897681662, max timestamp=1733897681673Num uniques 1136; 2024-12-11T06:14:41,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-11T06:14:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897681662, max timestamp=1733897681673Num uniques 1136; 2024-12-11T06:14:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897681662, max timestamp=1733897681673Num uniques 1136; 2024-12-11T06:14:41,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-11T06:14:41,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-12-11T06:14:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733897681696, max timestamp=1733897681714Num uniques 1145; 2024-12-11T06:14:41,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-11T06:14:41,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733897681701, max timestamp=1733897681714Num uniques 1145; 2024-12-11T06:14:41,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733897681701, max timestamp=1733897681714Num uniques 1145; 2024-12-11T06:14:41,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-11T06:14:41,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-11T06:14:41,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733897681755, max timestamp=1733897681777Num uniques 1104; 2024-12-11T06:14:41,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733897681755, max timestamp=1733897681777Num uniques 1104; 2024-12-11T06:14:41,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=404, cellSize=18500, totalHeapSize=42256, min timestamp=1733897681755, max timestamp=1733897681777Num uniques 1154; 2024-12-11T06:14:41,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=18500, suffix heap size=147984,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=404 2024-12-11T06:14:41,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=18500, suffix heap size=142784,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=404 2024-12-11T06:14:41,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=18500, suffix heap size=142784,new segment heap size=42256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=404 2024-12-11T06:14:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897741808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:41,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897741813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897741813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897741814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897741938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897741938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897741938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897741938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:41,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 2024-12-11T06:14:41,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/df25b9102bb74246a01001f245231b46 is 50, key is test_row_0/C:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741917_1093 (size=12301) 2024-12-11T06:14:41,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:41,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897742148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897742148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897742148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897742149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:42,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:42,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/df25b9102bb74246a01001f245231b46 2024-12-11T06:14:42,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/72db22ca37434560b94a1e5d7f86ec32 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32 2024-12-11T06:14:42,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32, entries=150, sequenceid=343, filesize=12.0 K 2024-12-11T06:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 2024-12-11T06:14:42,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2, entries=150, sequenceid=343, filesize=12.0 K 2024-12-11T06:14:42,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/df25b9102bb74246a01001f245231b46 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46 2024-12-11T06:14:42,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46, entries=150, sequenceid=343, filesize=12.0 K 2024-12-11T06:14:42,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=167.72 KB/171750 for 54b3d700051ccb5e9981f4a99fd0aae5 in 943ms, sequenceid=343, compaction requested=true 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:42,431 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:42,431 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:42,440 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:42,440 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:42,440 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,441 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/fc228c8c52ef4939925fff6ff9004b9c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:42,441 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:42,441 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:42,441 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,441 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/aef1f3308ffb4ba686ff4d0b0927a837, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:42,442 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fc228c8c52ef4939925fff6ff9004b9c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:42,442 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef1f3308ffb4ba686ff4d0b0927a837, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:42,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,443 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d820fa1bd343e3ad93bd8fcc6b27a4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733897678664 2024-12-11T06:14:42,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=167.72 KB heapSize=461.63 KB 2024-12-11T06:14:42,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33430614689482380a00896428d41a0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733897678664 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:42,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72db22ca37434560b94a1e5d7f86ec32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:42,443 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3b07fb2f8d46ffbe02e33f3f98eda2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:42,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/7ecbc7230b6b478ab20500fc6c463a78 is 50, key is test_row_0/A:col10/1733897681808/Put/seqid=0 2024-12-11T06:14:42,462 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:42,462 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/4ac92912f5a1432a803fc83f1f4f052a is 50, key is test_row_0/A:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:42,463 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:42,464 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e40308f6e7c042278862d2970c59add3 is 50, key is test_row_0/B:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:42,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:42,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897742462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897742470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897742471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897742473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741918_1094 (size=13221) 2024-12-11T06:14:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741919_1095 (size=13221) 2024-12-11T06:14:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741920_1096 (size=12301) 2024-12-11T06:14:42,493 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/7ecbc7230b6b478ab20500fc6c463a78 2024-12-11T06:14:42,514 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/e40308f6e7c042278862d2970c59add3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e40308f6e7c042278862d2970c59add3 2024-12-11T06:14:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/ffe7497136e0423ba4b76e9ad73837f2 is 50, key is test_row_0/B:col10/1733897681808/Put/seqid=0 2024-12-11T06:14:42,527 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into e40308f6e7c042278862d2970c59add3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:42,527 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:42,527 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897682431; duration=0sec 2024-12-11T06:14:42,527 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:42,527 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:42,527 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:42,529 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:42,529 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:42,529 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:42,529 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7e3301da426c4ce993f987581b3199d3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:42,530 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e3301da426c4ce993f987581b3199d3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733897678328 2024-12-11T06:14:42,530 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e96f0b5c2439490f8db80baf719b8573, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733897678664 2024-12-11T06:14:42,530 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting df25b9102bb74246a01001f245231b46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:42,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741921_1097 (size=12301) 2024-12-11T06:14:42,546 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:42,547 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/5f715079275741beb52c5a633535f343 is 50, key is test_row_0/C:col10/1733897680853/Put/seqid=0 2024-12-11T06:14:42,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741922_1098 (size=13221) 2024-12-11T06:14:42,585 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/5f715079275741beb52c5a633535f343 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/5f715079275741beb52c5a633535f343 2024-12-11T06:14:42,594 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into 5f715079275741beb52c5a633535f343(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:42,594 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:42,594 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897682431; duration=0sec 2024-12-11T06:14:42,594 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:42,594 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897742589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897742589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897742589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:42,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897742801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897742809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897742809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,905 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/4ac92912f5a1432a803fc83f1f4f052a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4ac92912f5a1432a803fc83f1f4f052a 2024-12-11T06:14:42,912 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 4ac92912f5a1432a803fc83f1f4f052a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:42,912 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:42,913 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897682431; duration=0sec 2024-12-11T06:14:42,913 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:42,913 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:42,943 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/ffe7497136e0423ba4b76e9ad73837f2 2024-12-11T06:14:42,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c451aa5992be460f88b0ca5ddd29e73c is 50, key is test_row_0/C:col10/1733897681808/Put/seqid=0 2024-12-11T06:14:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897742988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:42,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741923_1099 (size=12301) 2024-12-11T06:14:43,004 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c451aa5992be460f88b0ca5ddd29e73c 2024-12-11T06:14:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/7ecbc7230b6b478ab20500fc6c463a78 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78 2024-12-11T06:14:43,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78, entries=150, sequenceid=466, filesize=12.0 K 2024-12-11T06:14:43,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/ffe7497136e0423ba4b76e9ad73837f2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2 2024-12-11T06:14:43,119 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2, entries=150, sequenceid=466, filesize=12.0 K 2024-12-11T06:14:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c451aa5992be460f88b0ca5ddd29e73c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c 2024-12-11T06:14:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897743119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897743119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897743121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,142 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c, entries=150, sequenceid=466, filesize=12.0 K 2024-12-11T06:14:43,144 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~461.58 KB/472656, currentSize=20.13 KB/20610 for 54b3d700051ccb5e9981f4a99fd0aae5 in 700ms, sequenceid=466, compaction requested=false 2024-12-11T06:14:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-11T06:14:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-11T06:14:43,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-11T06:14:43,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-12-11T06:14:43,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6540 sec 2024-12-11T06:14:43,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T06:14:43,661 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:43,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-11T06:14:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T06:14:43,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/28af96ac703d46d79b728b00f89c6fb8 is 50, key is test_row_0/A:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:43,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:43,683 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:43,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:43,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741924_1100 (size=9857) 2024-12-11T06:14:43,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/28af96ac703d46d79b728b00f89c6fb8 2024-12-11T06:14:43,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897683673, max timestamp=1733897683708Num uniques 859; 2024-12-11T06:14:43,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:14:43,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897683673, max timestamp=1733897683708Num uniques 859; 2024-12-11T06:14:43,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:14:43,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897683673, max timestamp=1733897683708Num uniques 859; 2024-12-11T06:14:43,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:14:43,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/75d41d49b4d54f6284abca908b1a05c3 is 50, key is test_row_0/B:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:43,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741925_1101 (size=9857) 2024-12-11T06:14:43,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/75d41d49b4d54f6284abca908b1a05c3 2024-12-11T06:14:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T06:14:43,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897743789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897743792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897743793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/75ceee8bf5b643b6b8fec916c9d12071 is 50, key is test_row_0/C:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T06:14:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:43,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:43,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:43,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741926_1102 (size=9857) 2024-12-11T06:14:43,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/75ceee8bf5b643b6b8fec916c9d12071 2024-12-11T06:14:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/28af96ac703d46d79b728b00f89c6fb8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8 2024-12-11T06:14:43,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897743899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8, entries=100, sequenceid=479, filesize=9.6 K 2024-12-11T06:14:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/75d41d49b4d54f6284abca908b1a05c3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3 2024-12-11T06:14:43,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897743919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897743929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:43,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3, entries=100, sequenceid=479, filesize=9.6 K 2024-12-11T06:14:43,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/75ceee8bf5b643b6b8fec916c9d12071 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071 2024-12-11T06:14:43,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071, entries=100, sequenceid=479, filesize=9.6 K 2024-12-11T06:14:43,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 54b3d700051ccb5e9981f4a99fd0aae5 in 297ms, sequenceid=479, compaction requested=true 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:43,946 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:43,946 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:43,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:43,948 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:43,948 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:43,948 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:43,948 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4ac92912f5a1432a803fc83f1f4f052a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:43,948 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:43,948 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:43,948 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:43,949 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e40308f6e7c042278862d2970c59add3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:43,950 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac92912f5a1432a803fc83f1f4f052a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:43,950 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e40308f6e7c042278862d2970c59add3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:43,950 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecbc7230b6b478ab20500fc6c463a78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733897681796 2024-12-11T06:14:43,950 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe7497136e0423ba4b76e9ad73837f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733897681796 2024-12-11T06:14:43,951 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28af96ac703d46d79b728b00f89c6fb8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897683642 2024-12-11T06:14:43,951 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d41d49b4d54f6284abca908b1a05c3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897683642 2024-12-11T06:14:43,989 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:43,990 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/56dd2b897c424ad4b31606678066b0d2 is 50, key is test_row_0/B:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:44,001 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:44,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T06:14:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:44,001 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-11T06:14:44,001 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/24c0f9aac1444c4f9dd7f73a1b7f764d is 50, key is test_row_0/A:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:44,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:44,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:44,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:44,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:14:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T06:14:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:44,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. as already flushing 2024-12-11T06:14:44,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741927_1103 (size=13323) 2024-12-11T06:14:44,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741928_1104 (size=13323) 2024-12-11T06:14:44,044 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/56dd2b897c424ad4b31606678066b0d2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/56dd2b897c424ad4b31606678066b0d2 2024-12-11T06:14:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3010f8e98e1b4d2baa300baf160d7378 is 50, key is test_row_0/A:col10/1733897683786/Put/seqid=0 2024-12-11T06:14:44,058 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/24c0f9aac1444c4f9dd7f73a1b7f764d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/24c0f9aac1444c4f9dd7f73a1b7f764d 2024-12-11T06:14:44,077 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 56dd2b897c424ad4b31606678066b0d2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:44,077 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:44,077 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897683946; duration=0sec 2024-12-11T06:14:44,077 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:44,078 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:44,078 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741929_1105 (size=12301) 2024-12-11T06:14:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897744078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,089 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:44,089 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:44,089 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:44,089 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3010f8e98e1b4d2baa300baf160d7378 2024-12-11T06:14:44,089 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/5f715079275741beb52c5a633535f343, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=34.5 K 2024-12-11T06:14:44,090 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 24c0f9aac1444c4f9dd7f73a1b7f764d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:44,090 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:44,090 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897683946; duration=0sec 2024-12-11T06:14:44,090 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:44,090 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:44,096 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f715079275741beb52c5a633535f343, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733897680851 2024-12-11T06:14:44,101 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c451aa5992be460f88b0ca5ddd29e73c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733897681796 2024-12-11T06:14:44,109 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ceee8bf5b643b6b8fec916c9d12071, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897683642 2024-12-11T06:14:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/1b8a42ddb746483faf6a3be72ffbacc8 is 50, key is test_row_0/B:col10/1733897683786/Put/seqid=0 2024-12-11T06:14:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897744112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,134 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:44,135 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c1057c155b924f93b620d3b62a4bb144 is 50, key is test_row_0/C:col10/1733897683642/Put/seqid=0 2024-12-11T06:14:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897744140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897744145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741930_1106 (size=12301) 2024-12-11T06:14:44,163 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/1b8a42ddb746483faf6a3be72ffbacc8 2024-12-11T06:14:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/bc22f49dba1a4175bad75cca8144cdd6 is 50, key is test_row_0/C:col10/1733897683786/Put/seqid=0 2024-12-11T06:14:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897744193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741931_1107 (size=13323) 2024-12-11T06:14:44,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741932_1108 (size=12301) 2024-12-11T06:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T06:14:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46228 deadline: 1733897744409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46212 deadline: 1733897744439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46236 deadline: 1733897744459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46202 deadline: 1733897744459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,476 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-11T06:14:44,477 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,478 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-11T06:14:44,478 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46166 deadline: 1733897744478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:44,479 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-11T06:14:44,479 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,479 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 54b3d700051ccb5e9981f4a99fd0aae5 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18373 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:14:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:14:44,482 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-11T06:14:44,483 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,484 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,484 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,484 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,485 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,484 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,484 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,484 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,611 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/c1057c155b924f93b620d3b62a4bb144 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c1057c155b924f93b620d3b62a4bb144 2024-12-11T06:14:44,617 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into c1057c155b924f93b620d3b62a4bb144(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:44,617 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:44,617 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897683946; duration=0sec 2024-12-11T06:14:44,617 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:44,618 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:44,627 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/bc22f49dba1a4175bad75cca8144cdd6 2024-12-11T06:14:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/3010f8e98e1b4d2baa300baf160d7378 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378 2024-12-11T06:14:44,638 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378, entries=150, sequenceid=515, filesize=12.0 K 2024-12-11T06:14:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/1b8a42ddb746483faf6a3be72ffbacc8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8 2024-12-11T06:14:44,645 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8, entries=150, sequenceid=515, filesize=12.0 K 2024-12-11T06:14:44,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/bc22f49dba1a4175bad75cca8144cdd6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6 2024-12-11T06:14:44,651 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6, entries=150, sequenceid=515, filesize=12.0 K 2024-12-11T06:14:44,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 54b3d700051ccb5e9981f4a99fd0aae5 in 652ms, sequenceid=515, compaction requested=false 2024-12-11T06:14:44,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:44,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:44,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-11T06:14:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-11T06:14:44,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-11T06:14:44,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 972 msec 2024-12-11T06:14:44,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 993 msec 2024-12-11T06:14:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:44,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:14:44,730 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-11T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:44,730 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,730 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:44,730 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:44,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:44,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6662509045e84ef09ac65ada03b777ae is 50, key is test_row_0/A:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:44,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741933_1109 (size=12301) 2024-12-11T06:14:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T06:14:44,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:44,950 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-11T06:14:44,951 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,951 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,951 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,971 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-11T06:14:44,971 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,971 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,971 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:44,980 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-11T06:14:44,980 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:44,980 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,980 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:44,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:45,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6662509045e84ef09ac65ada03b777ae 2024-12-11T06:14:45,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/0bc2838047c44c3194649b5a0f23cf65 is 50, key is test_row_0/B:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741934_1110 (size=12301) 2024-12-11T06:14:45,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/0bc2838047c44c3194649b5a0f23cf65 2024-12-11T06:14:45,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd is 50, key is test_row_0/C:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741935_1111 (size=12301) 2024-12-11T06:14:45,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd 2024-12-11T06:14:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/6662509045e84ef09ac65ada03b777ae as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae 2024-12-11T06:14:45,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae, entries=150, sequenceid=529, filesize=12.0 K 2024-12-11T06:14:45,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/0bc2838047c44c3194649b5a0f23cf65 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65 2024-12-11T06:14:45,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65, entries=150, sequenceid=529, filesize=12.0 K 2024-12-11T06:14:45,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd 2024-12-11T06:14:45,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd, entries=150, sequenceid=529, filesize=12.0 K 2024-12-11T06:14:45,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 54b3d700051ccb5e9981f4a99fd0aae5 in 471ms, sequenceid=529, compaction requested=true 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:45,202 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:14:45,202 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54b3d700051ccb5e9981f4a99fd0aae5:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:14:45,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:45,204 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:45,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:45,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/B is initiating minor compaction (all files) 2024-12-11T06:14:45,204 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/A is initiating minor compaction (all files) 2024-12-11T06:14:45,204 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/A in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:45,204 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/B in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:45,204 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/56dd2b897c424ad4b31606678066b0d2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=37.0 K 2024-12-11T06:14:45,204 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/24c0f9aac1444c4f9dd7f73a1b7f764d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=37.0 K 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 56dd2b897c424ad4b31606678066b0d2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897681796 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24c0f9aac1444c4f9dd7f73a1b7f764d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897681796 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8a42ddb746483faf6a3be72ffbacc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733897683771 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3010f8e98e1b4d2baa300baf160d7378, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733897683771 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc2838047c44c3194649b5a0f23cf65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1733897684064 2024-12-11T06:14:45,205 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6662509045e84ef09ac65ada03b777ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1733897684064 2024-12-11T06:14:45,217 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:45,217 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/93945825e7a84444b8aecadf2c5dcf7d is 50, key is test_row_0/B:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:45,224 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#A#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:45,225 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0f7c6d21002047d787d34af28eecb21c is 50, key is test_row_0/A:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:45,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741936_1112 (size=13425) 2024-12-11T06:14:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741937_1113 (size=13425) 2024-12-11T06:14:45,239 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/93945825e7a84444b8aecadf2c5dcf7d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/93945825e7a84444b8aecadf2c5dcf7d 2024-12-11T06:14:45,244 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/0f7c6d21002047d787d34af28eecb21c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7c6d21002047d787d34af28eecb21c 2024-12-11T06:14:45,248 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/B of 54b3d700051ccb5e9981f4a99fd0aae5 into 93945825e7a84444b8aecadf2c5dcf7d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:45,248 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:45,248 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/B, priority=13, startTime=1733897685202; duration=0sec 2024-12-11T06:14:45,248 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:14:45,248 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:B 2024-12-11T06:14:45,249 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:14:45,250 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:14:45,250 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 54b3d700051ccb5e9981f4a99fd0aae5/C is initiating minor compaction (all files) 2024-12-11T06:14:45,250 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54b3d700051ccb5e9981f4a99fd0aae5/C in TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:45,250 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c1057c155b924f93b620d3b62a4bb144, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp, totalSize=37.0 K 2024-12-11T06:14:45,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c1057c155b924f93b620d3b62a4bb144, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1733897681796 2024-12-11T06:14:45,252 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bc22f49dba1a4175bad75cca8144cdd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1733897683771 2024-12-11T06:14:45,252 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3ec38fa30c4d78a7fb1ed4d1160ccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1733897684064 2024-12-11T06:14:45,253 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/A of 54b3d700051ccb5e9981f4a99fd0aae5 into 0f7c6d21002047d787d34af28eecb21c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:45,253 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:45,253 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/A, priority=13, startTime=1733897685202; duration=0sec 2024-12-11T06:14:45,253 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:45,253 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:A 2024-12-11T06:14:45,263 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54b3d700051ccb5e9981f4a99fd0aae5#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:14:45,264 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/af960fea13db4818803c5a2fdf1c5ab1 is 50, key is test_row_0/C:col10/1733897684069/Put/seqid=0 2024-12-11T06:14:45,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741938_1114 (size=13425) 2024-12-11T06:14:45,674 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/af960fea13db4818803c5a2fdf1c5ab1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/af960fea13db4818803c5a2fdf1c5ab1 2024-12-11T06:14:45,679 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54b3d700051ccb5e9981f4a99fd0aae5/C of 54b3d700051ccb5e9981f4a99fd0aae5 into af960fea13db4818803c5a2fdf1c5ab1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:14:45,679 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54b3d700051ccb5e9981f4a99fd0aae5: 2024-12-11T06:14:45,679 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., storeName=54b3d700051ccb5e9981f4a99fd0aae5/C, priority=13, startTime=1733897685202; duration=0sec 2024-12-11T06:14:45,679 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:14:45,679 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54b3d700051ccb5e9981f4a99fd0aae5:C 2024-12-11T06:14:46,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T06:14:54,510 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-11T06:14:54,510 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:14:54,510 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:54,510 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:54,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3006 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2984 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1411 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4223 rows 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1438 2024-12-11T06:14:54,511 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4307 rows 2024-12-11T06:14:54,512 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:14:54,512 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:14:54,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:54,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:54,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:14:54,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:14:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:54,524 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897694524"}]},"ts":"1733897694524"} 2024-12-11T06:14:54,526 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:14:54,527 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:14:54,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:14:54,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, UNASSIGN}] 2024-12-11T06:14:54,534 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, UNASSIGN 2024-12-11T06:14:54,535 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=54b3d700051ccb5e9981f4a99fd0aae5, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:54,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, UNASSIGN because future has completed 2024-12-11T06:14:54,538 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:14:54,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:54,694 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:54,694 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:14:54,695 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 54b3d700051ccb5e9981f4a99fd0aae5, disabling compactions & flushes 2024-12-11T06:14:54,695 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:54,695 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:54,695 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. after waiting 0 ms 2024-12-11T06:14:54,695 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:54,695 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 54b3d700051ccb5e9981f4a99fd0aae5 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=A 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=B 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54b3d700051ccb5e9981f4a99fd0aae5, store=C 2024-12-11T06:14:54,696 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:54,703 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f8a6f3dcd46a49e6adc63ce64e912bdd is 50, key is test_row_1/A:col10/1733897684979/Put/seqid=0 2024-12-11T06:14:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741939_1115 (size=9857) 2024-12-11T06:14:54,724 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f8a6f3dcd46a49e6adc63ce64e912bdd 2024-12-11T06:14:54,733 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/8e853057c68845d2963586058ce7f068 is 50, key is test_row_1/B:col10/1733897684979/Put/seqid=0 2024-12-11T06:14:54,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741940_1116 (size=9857) 2024-12-11T06:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:55,138 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/8e853057c68845d2963586058ce7f068 2024-12-11T06:14:55,146 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/a3efa445d1514e30910ee6ad93e06f2e is 50, key is test_row_1/C:col10/1733897684979/Put/seqid=0 2024-12-11T06:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741941_1117 (size=9857) 2024-12-11T06:14:55,552 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/a3efa445d1514e30910ee6ad93e06f2e 2024-12-11T06:14:55,558 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/A/f8a6f3dcd46a49e6adc63ce64e912bdd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f8a6f3dcd46a49e6adc63ce64e912bdd 2024-12-11T06:14:55,564 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f8a6f3dcd46a49e6adc63ce64e912bdd, entries=100, sequenceid=539, filesize=9.6 K 2024-12-11T06:14:55,565 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/B/8e853057c68845d2963586058ce7f068 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/8e853057c68845d2963586058ce7f068 2024-12-11T06:14:55,572 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/8e853057c68845d2963586058ce7f068, entries=100, sequenceid=539, filesize=9.6 K 2024-12-11T06:14:55,573 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/.tmp/C/a3efa445d1514e30910ee6ad93e06f2e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/a3efa445d1514e30910ee6ad93e06f2e 2024-12-11T06:14:55,579 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/a3efa445d1514e30910ee6ad93e06f2e, entries=100, sequenceid=539, filesize=9.6 K 2024-12-11T06:14:55,580 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 54b3d700051ccb5e9981f4a99fd0aae5 in 885ms, sequenceid=539, compaction requested=false 2024-12-11T06:14:55,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/21c94f8f1d9e419897f2c9c7c043f5fb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0dfeb9225da44b9192c9c4fe2aa36f70, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7744fc70d44dfcae93fb2fe37150be, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/af18ccfdaead40c3a46555a44c7c659d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/964e542c6e3748c4bf16d7fb0e977b9d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e7fa7486262741098c703cf78c184d30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b54a09ae4f624464b18b198d3acf688e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/aef1f3308ffb4ba686ff4d0b0927a837, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4ac92912f5a1432a803fc83f1f4f052a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/24c0f9aac1444c4f9dd7f73a1b7f764d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae] to archive 2024-12-11T06:14:55,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:14:55,593 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5af0f39fc6044b2f83980a28f9df339d 2024-12-11T06:14:55,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/5f0c0814aef740b9bd9ecb685c394ad7 2024-12-11T06:14:55,598 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/dfe893ff268647ecb9e5f3c6bac23e17 2024-12-11T06:14:55,598 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6c66c66ea63d49618c6ad2865c12a06d 2024-12-11T06:14:55,600 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/36df3419ae9d4cac8ac0eb41a7400f3a 2024-12-11T06:14:55,600 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/fa08b84d13a348ae9d135518bd8a5280 2024-12-11T06:14:55,600 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4bb774c0646944e09540dcae35c26747 2024-12-11T06:14:55,600 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3119be60f4d6478ab31ff89a62828f5f 2024-12-11T06:14:55,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7744fc70d44dfcae93fb2fe37150be to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7744fc70d44dfcae93fb2fe37150be 2024-12-11T06:14:55,601 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0dfeb9225da44b9192c9c4fe2aa36f70 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0dfeb9225da44b9192c9c4fe2aa36f70 2024-12-11T06:14:55,601 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/11a93b3b76b74176ad5af7644215f525 2024-12-11T06:14:55,602 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/af18ccfdaead40c3a46555a44c7c659d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/af18ccfdaead40c3a46555a44c7c659d 2024-12-11T06:14:55,604 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/21c94f8f1d9e419897f2c9c7c043f5fb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/21c94f8f1d9e419897f2c9c7c043f5fb 2024-12-11T06:14:55,604 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/441e52c29eaf4474ad1ad4ef7ebdee27 2024-12-11T06:14:55,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/964e542c6e3748c4bf16d7fb0e977b9d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/964e542c6e3748c4bf16d7fb0e977b9d 2024-12-11T06:14:55,605 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b20ccb90ff4541d490762c5df649f6d4 2024-12-11T06:14:55,605 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e7fa7486262741098c703cf78c184d30 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e7fa7486262741098c703cf78c184d30 2024-12-11T06:14:55,605 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/afd379e83d3d4fc2a3f82d8649d5abff 2024-12-11T06:14:55,614 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/82a878eb952d40769f85a585a7785cb5 2024-12-11T06:14:55,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8f24e1ff7b3743a8926de93b783a3eb4 2024-12-11T06:14:55,618 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b54a09ae4f624464b18b198d3acf688e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/b54a09ae4f624464b18b198d3acf688e 2024-12-11T06:14:55,618 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/e33430614689482380a00896428d41a0 2024-12-11T06:14:55,619 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f93b32b058914616ad50bb1a959b26fc 2024-12-11T06:14:55,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/8a68126706534070928afa01d4bf6d1b 2024-12-11T06:14:55,619 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3b36c65a6c404ee08a1f7608939cdeb8 2024-12-11T06:14:55,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/72db22ca37434560b94a1e5d7f86ec32 2024-12-11T06:14:55,620 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/aef1f3308ffb4ba686ff4d0b0927a837 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/aef1f3308ffb4ba686ff4d0b0927a837 2024-12-11T06:14:55,620 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/7ecbc7230b6b478ab20500fc6c463a78 2024-12-11T06:14:55,622 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4ac92912f5a1432a803fc83f1f4f052a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/4ac92912f5a1432a803fc83f1f4f052a 2024-12-11T06:14:55,622 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/6662509045e84ef09ac65ada03b777ae 2024-12-11T06:14:55,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/3010f8e98e1b4d2baa300baf160d7378 2024-12-11T06:14:55,622 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/24c0f9aac1444c4f9dd7f73a1b7f764d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/24c0f9aac1444c4f9dd7f73a1b7f764d 2024-12-11T06:14:55,623 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/28af96ac703d46d79b728b00f89c6fb8 2024-12-11T06:14:55,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/2f223b9ddd9f4f169185ac9b3e39175e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/df729464ee3e42dbb04308b60ec12124, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1767c2f37c55430589bb8e50d136efc8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c5ebe7147c014e228677cad9f0e06faa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/62160f255f47497ca1f51d9f8080d7c8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a23d609c92241c48205d6649e65b03d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/fc228c8c52ef4939925fff6ff9004b9c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e40308f6e7c042278862d2970c59add3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/56dd2b897c424ad4b31606678066b0d2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65] to archive 2024-12-11T06:14:55,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:14:55,647 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a2acbe6bdeba4a1a9dc5c22fcdd2eae6 2024-12-11T06:14:55,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c00d39e3e7dd4fdba93988ae8bcd46f7 2024-12-11T06:14:55,647 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/6a1fee743fe841509a46bf9b182a0582 2024-12-11T06:14:55,647 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/7b4ccb9e0e7c4ac9b5f5bc1da1edb152 2024-12-11T06:14:55,647 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f8ef82ac95594a98ad1b26092fc5884f 2024-12-11T06:14:55,648 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/b88155bc852d4947b13d558f629c4d01 2024-12-11T06:14:55,648 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/a717b59e15da4d81a6c013a17e216fb8 2024-12-11T06:14:55,650 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/2f223b9ddd9f4f169185ac9b3e39175e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/2f223b9ddd9f4f169185ac9b3e39175e 2024-12-11T06:14:55,651 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/df729464ee3e42dbb04308b60ec12124 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/df729464ee3e42dbb04308b60ec12124 2024-12-11T06:14:55,652 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/5f5f224fb15549fea33943675b99697f 2024-12-11T06:14:55,652 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e7238b66b416423580f3a23751c3ac1b 2024-12-11T06:14:55,652 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/f140dbb6d26e48fb9e2fd40f2abb0c94 2024-12-11T06:14:55,652 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a6e02a457544b11a7e5f1d2c8683744 2024-12-11T06:14:55,653 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c5ebe7147c014e228677cad9f0e06faa to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c5ebe7147c014e228677cad9f0e06faa 2024-12-11T06:14:55,653 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72b8b9926b37443e966def0c080956e4 2024-12-11T06:14:55,654 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a23d609c92241c48205d6649e65b03d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9a23d609c92241c48205d6649e65b03d 2024-12-11T06:14:55,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/669990fa4a9943389bf3ced20954fd72 2024-12-11T06:14:55,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e01bdc312add47f2b2866d7d367b5904 2024-12-11T06:14:55,656 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1767c2f37c55430589bb8e50d136efc8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1767c2f37c55430589bb8e50d136efc8 2024-12-11T06:14:55,656 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/62160f255f47497ca1f51d9f8080d7c8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/62160f255f47497ca1f51d9f8080d7c8 2024-12-11T06:14:55,657 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e032e7930a914e7c9cb39c96deb8853d 2024-12-11T06:14:55,657 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/82ae7d82eb7b442585d3053e4ac46908 2024-12-11T06:14:55,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/c1cc645cd82d4b97a5e85f85e55ad8e6 2024-12-11T06:14:55,659 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/9dcd40d6054d4793a60b3378cc1754e1 2024-12-11T06:14:55,659 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/fc228c8c52ef4939925fff6ff9004b9c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/fc228c8c52ef4939925fff6ff9004b9c 2024-12-11T06:14:55,660 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e40308f6e7c042278862d2970c59add3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/e40308f6e7c042278862d2970c59add3 2024-12-11T06:14:55,660 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/72d820fa1bd343e3ad93bd8fcc6b27a4 2024-12-11T06:14:55,660 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/bd3b07fb2f8d46ffbe02e33f3f98eda2 2024-12-11T06:14:55,660 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/ffe7497136e0423ba4b76e9ad73837f2 2024-12-11T06:14:55,661 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/56dd2b897c424ad4b31606678066b0d2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/56dd2b897c424ad4b31606678066b0d2 2024-12-11T06:14:55,662 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/0bc2838047c44c3194649b5a0f23cf65 2024-12-11T06:14:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:55,662 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/75d41d49b4d54f6284abca908b1a05c3 2024-12-11T06:14:55,663 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/1b8a42ddb746483faf6a3be72ffbacc8 2024-12-11T06:14:55,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e04b472f0a91424ba5d4fc2bb47581fa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3102fa6b38414cc6a5a388924da2ebb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/cbc422d32ab04eb289f5251891d23dc9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/70f0e8d28fa546c8b8075a29ce378851, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/91de2bb8c90c455ab5533415ca987f71, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e491fd9f59c54f0185f4ba66d6cbefc3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/96ff66d3d78844da801b2da848509090, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7e3301da426c4ce993f987581b3199d3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/5f715079275741beb52c5a633535f343, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c1057c155b924f93b620d3b62a4bb144, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd] to archive 2024-12-11T06:14:55,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:14:55,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7f4580c32bc74bd3b363a8a3694a4f66 2024-12-11T06:14:55,670 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2341fd1ac6854a269836ffccb9bd8c13 2024-12-11T06:14:55,670 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2a48a7406fb54a89bae0bdb7e4f50c4f 2024-12-11T06:14:55,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/74d0d9b5fd8f4a59844460e95fffc66f 2024-12-11T06:14:55,671 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3102fa6b38414cc6a5a388924da2ebb6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3102fa6b38414cc6a5a388924da2ebb6 2024-12-11T06:14:55,671 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/523c26ea94964402a4ce7397c03e7e30 2024-12-11T06:14:55,671 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/be466ba4bb174afda24161ebb7a0b030 2024-12-11T06:14:55,671 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e04b472f0a91424ba5d4fc2bb47581fa to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e04b472f0a91424ba5d4fc2bb47581fa 2024-12-11T06:14:55,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/70f0e8d28fa546c8b8075a29ce378851 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/70f0e8d28fa546c8b8075a29ce378851 2024-12-11T06:14:55,675 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3fad3e12bc724f858707fa2fdce7ec5f 2024-12-11T06:14:55,675 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/8d6ec466db454b63b7ce98c114d5387b 2024-12-11T06:14:55,676 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/46b5b6c72c81404081d5558b804e8e31 2024-12-11T06:14:55,677 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/d71e78f3c3974f9a9165e8eb43779718 2024-12-11T06:14:55,677 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/cbc422d32ab04eb289f5251891d23dc9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/cbc422d32ab04eb289f5251891d23dc9 2024-12-11T06:14:55,677 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/91de2bb8c90c455ab5533415ca987f71 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/91de2bb8c90c455ab5533415ca987f71 2024-12-11T06:14:55,679 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c3ea842473714c959368feac6d1375e4 2024-12-11T06:14:55,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/391e4e644a03469dac5ae23466cbc43e 2024-12-11T06:14:55,679 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e491fd9f59c54f0185f4ba66d6cbefc3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e491fd9f59c54f0185f4ba66d6cbefc3 2024-12-11T06:14:55,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/39b00a3c47ad4169b0df19771cfaf265 2024-12-11T06:14:55,680 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/773f4b6568514dfb91a307fcef4d7a03 2024-12-11T06:14:55,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f5339688a806457ebdedf2a5281749cc 2024-12-11T06:14:55,681 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/96ff66d3d78844da801b2da848509090 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/96ff66d3d78844da801b2da848509090 2024-12-11T06:14:55,682 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7e3301da426c4ce993f987581b3199d3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/7e3301da426c4ce993f987581b3199d3 2024-12-11T06:14:55,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/e96f0b5c2439490f8db80baf719b8573 2024-12-11T06:14:55,684 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/df25b9102bb74246a01001f245231b46 2024-12-11T06:14:55,684 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c451aa5992be460f88b0ca5ddd29e73c 2024-12-11T06:14:55,685 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/5f715079275741beb52c5a633535f343 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/5f715079275741beb52c5a633535f343 2024-12-11T06:14:55,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/f8f153ad9e374e7e92841bbf1596f805 2024-12-11T06:14:55,685 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c1057c155b924f93b620d3b62a4bb144 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/c1057c155b924f93b620d3b62a4bb144 2024-12-11T06:14:55,686 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/75ceee8bf5b643b6b8fec916c9d12071 2024-12-11T06:14:55,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/bc22f49dba1a4175bad75cca8144cdd6 2024-12-11T06:14:55,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/3c3ec38fa30c4d78a7fb1ed4d1160ccd 2024-12-11T06:14:55,689 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/2c20daec222a414aac9a03adc5ffd47b 2024-12-11T06:14:55,697 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=1 2024-12-11T06:14:55,698 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. 2024-12-11T06:14:55,698 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 54b3d700051ccb5e9981f4a99fd0aae5: Waiting for close lock at 1733897694695Running coprocessor pre-close hooks at 1733897694695Disabling compacts and flushes for region at 1733897694695Disabling writes for close at 1733897694695Obtaining lock to block concurrent updates at 1733897694695Preparing flush snapshotting stores in 54b3d700051ccb5e9981f4a99fd0aae5 at 1733897694695Finished memstore snapshotting TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733897694696 (+1 ms)Flushing stores of TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5. at 1733897694696Flushing 54b3d700051ccb5e9981f4a99fd0aae5/A: creating writer at 1733897694696Flushing 54b3d700051ccb5e9981f4a99fd0aae5/A: appending metadata at 1733897694702 (+6 ms)Flushing 54b3d700051ccb5e9981f4a99fd0aae5/A: closing flushed file at 1733897694702Flushing 54b3d700051ccb5e9981f4a99fd0aae5/B: creating writer at 1733897694730 (+28 ms)Flushing 54b3d700051ccb5e9981f4a99fd0aae5/B: appending metadata at 1733897694733 (+3 ms)Flushing 54b3d700051ccb5e9981f4a99fd0aae5/B: closing flushed file at 1733897694733Flushing 54b3d700051ccb5e9981f4a99fd0aae5/C: creating writer at 1733897695142 (+409 ms)Flushing 54b3d700051ccb5e9981f4a99fd0aae5/C: appending metadata at 1733897695145 (+3 ms)Flushing 54b3d700051ccb5e9981f4a99fd0aae5/C: closing flushed file at 1733897695145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65f040c6: reopening flushed file at 1733897695557 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2550446e: reopening flushed file at 1733897695564 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ae74776: reopening flushed file at 1733897695572 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 54b3d700051ccb5e9981f4a99fd0aae5 in 885ms, sequenceid=539, compaction requested=false at 1733897695580 (+8 ms)Writing region close event to WAL at 1733897695690 (+110 ms)Running coprocessor post-close hooks at 1733897695698 (+8 ms)Closed at 1733897695698 2024-12-11T06:14:55,701 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:55,702 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=54b3d700051ccb5e9981f4a99fd0aae5, regionState=CLOSED 2024-12-11T06:14:55,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:14:55,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-11T06:14:55,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 54b3d700051ccb5e9981f4a99fd0aae5, server=4518703b2c1d,35425,1733897658824 in 1.1670 sec 2024-12-11T06:14:55,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-11T06:14:55,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54b3d700051ccb5e9981f4a99fd0aae5, UNASSIGN in 1.1730 sec 2024-12-11T06:14:55,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-11T06:14:55,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-11T06:14:55,713 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897695712"}]},"ts":"1733897695712"} 2024-12-11T06:14:55,715 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:14:55,715 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:14:55,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1980 sec 2024-12-11T06:14:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T06:14:56,679 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:56,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:14:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,687 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,692 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T06:14:56,696 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:56,703 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/recovered.edits] 2024-12-11T06:14:56,710 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7c6d21002047d787d34af28eecb21c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/0f7c6d21002047d787d34af28eecb21c 2024-12-11T06:14:56,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f8a6f3dcd46a49e6adc63ce64e912bdd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/A/f8a6f3dcd46a49e6adc63ce64e912bdd 2024-12-11T06:14:56,721 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/8e853057c68845d2963586058ce7f068 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/8e853057c68845d2963586058ce7f068 2024-12-11T06:14:56,721 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/93945825e7a84444b8aecadf2c5dcf7d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/B/93945825e7a84444b8aecadf2c5dcf7d 2024-12-11T06:14:56,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/a3efa445d1514e30910ee6ad93e06f2e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/a3efa445d1514e30910ee6ad93e06f2e 2024-12-11T06:14:56,728 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/af960fea13db4818803c5a2fdf1c5ab1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/C/af960fea13db4818803c5a2fdf1c5ab1 2024-12-11T06:14:56,733 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/recovered.edits/542.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5/recovered.edits/542.seqid 2024-12-11T06:14:56,734 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/54b3d700051ccb5e9981f4a99fd0aae5 2024-12-11T06:14:56,736 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:14:56,742 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-11T06:14:56,758 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:14:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T06:14:56,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:14:56,805 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,805 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:14:56,806 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897696805"}]},"ts":"9223372036854775807"} 2024-12-11T06:14:56,810 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:14:56,810 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 54b3d700051ccb5e9981f4a99fd0aae5, NAME => 'TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:14:56,811 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:14:56,811 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897696811"}]},"ts":"9223372036854775807"} 2024-12-11T06:14:56,813 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:14:56,814 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:56,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 132 msec 2024-12-11T06:14:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-11T06:14:57,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:14:57,010 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:57,022 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=253 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1096262115_22 at /127.0.0.1:37568 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4518703b2c1d:35425-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=762 (was 731) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=229 (was 684) 2024-12-11T06:14:57,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=253, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=762, ProcessCount=11, AvailableMemoryMB=228 2024-12-11T06:14:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:14:57,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:57,039 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:14:57,039 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:57,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-11T06:14:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T06:14:57,041 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:14:57,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741942_1118 (size=1061) 2024-12-11T06:14:57,055 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:14:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741943_1119 (size=53) 2024-12-11T06:14:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T06:14:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d0766cad7d1740597098828001c23b4a, disabling compactions & flushes 2024-12-11T06:14:57,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. after waiting 0 ms 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0766cad7d1740597098828001c23b4a: Waiting for close lock at 1733897697467Disabling compacts and flushes for region at 1733897697467Disabling writes for close at 1733897697467Writing region close event to WAL at 1733897697467Closed at 1733897697467 2024-12-11T06:14:57,469 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:14:57,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897697469"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897697469"}]},"ts":"1733897697469"} 2024-12-11T06:14:57,473 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:14:57,474 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:14:57,474 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897697474"}]},"ts":"1733897697474"} 2024-12-11T06:14:57,476 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:14:57,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, ASSIGN}] 2024-12-11T06:14:57,478 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, ASSIGN 2024-12-11T06:14:57,479 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:14:57,630 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:57,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, ASSIGN because future has completed 2024-12-11T06:14:57,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T06:14:57,789 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,789 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:14:57,789 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,789 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:57,790 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,790 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,792 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,801 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:57,802 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName A 2024-12-11T06:14:57,802 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:57,804 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:57,804 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,805 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:57,806 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName B 2024-12-11T06:14:57,806 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:57,807 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:57,807 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,808 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:57,810 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName C 2024-12-11T06:14:57,810 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:57,811 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:57,811 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,811 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,814 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,814 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,816 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,816 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,816 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:14:57,818 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,828 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:14:57,828 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened d0766cad7d1740597098828001c23b4a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62173523, jitterRate=-0.07354231178760529}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:14:57,828 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:57,830 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d0766cad7d1740597098828001c23b4a: Running coprocessor pre-open hook at 1733897697790Writing region info on filesystem at 1733897697790Initializing all the Stores at 1733897697791 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897697791Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897697791Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897697791Cleaning up temporary data from old regions at 1733897697816 (+25 ms)Running coprocessor post-open hooks at 1733897697828 (+12 ms)Region opened successfully at 1733897697829 (+1 ms) 2024-12-11T06:14:57,830 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., pid=30, masterSystemTime=1733897697784 2024-12-11T06:14:57,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:57,836 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,836 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:57,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:14:57,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-11T06:14:57,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 in 226 msec 2024-12-11T06:14:57,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-11T06:14:57,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, ASSIGN in 384 msec 2024-12-11T06:14:57,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:14:57,865 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897697864"}]},"ts":"1733897697864"} 2024-12-11T06:14:57,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:14:57,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:14:57,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 835 msec 2024-12-11T06:14:58,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T06:14:58,181 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:58,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aab2206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:58,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:58,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:58,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4ce441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:58,192 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b0c527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:58,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:58,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:58,201 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:14:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:14:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5406e4ed 2024-12-11T06:14:58,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:14:58,204 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:14:58,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:14:58,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:14:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T06:14:58,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741944_1120 (size=1097) 2024-12-11T06:14:58,689 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T06:14:58,690 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T06:14:58,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:14:58,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE}] 2024-12-11T06:14:58,700 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE 2024-12-11T06:14:58,701 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE because future has completed 2024-12-11T06:14:58,703 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:14:58,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:58,856 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:58,856 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:14:58,856 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing d0766cad7d1740597098828001c23b4a, disabling compactions & flushes 2024-12-11T06:14:58,856 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:58,856 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:58,856 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. after waiting 0 ms 2024-12-11T06:14:58,856 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:58,859 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T06:14:58,860 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:58,860 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for d0766cad7d1740597098828001c23b4a: Waiting for close lock at 1733897698856Running coprocessor pre-close hooks at 1733897698856Disabling compacts and flushes for region at 1733897698856Disabling writes for close at 1733897698856Writing region close event to WAL at 1733897698856Running coprocessor post-close hooks at 1733897698860 (+4 ms)Closed at 1733897698860 2024-12-11T06:14:58,860 WARN [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: d0766cad7d1740597098828001c23b4a to self. 2024-12-11T06:14:58,862 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:58,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=CLOSED 2024-12-11T06:14:58,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:14:58,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-11T06:14:58,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 in 163 msec 2024-12-11T06:14:58,870 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE; state=CLOSED, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=true 2024-12-11T06:14:59,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE because future has completed 2024-12-11T06:14:59,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:14:59,179 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,180 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:14:59,180 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,180 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:14:59,180 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,180 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,185 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,186 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:59,191 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName A 2024-12-11T06:14:59,194 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:59,194 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:59,195 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,196 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:59,196 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName B 2024-12-11T06:14:59,196 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:59,197 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:59,197 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,198 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:14:59,198 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0766cad7d1740597098828001c23b4a columnFamilyName C 2024-12-11T06:14:59,198 DEBUG [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:59,199 INFO [StoreOpener-d0766cad7d1740597098828001c23b4a-1 {}] regionserver.HStore(327): Store=d0766cad7d1740597098828001c23b4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:14:59,199 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,199 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,200 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,203 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,204 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,204 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,204 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:14:59,206 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,207 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened d0766cad7d1740597098828001c23b4a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73469750, jitterRate=0.09478458762168884}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:14:59,207 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,208 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for d0766cad7d1740597098828001c23b4a: Running coprocessor pre-open hook at 1733897699180Writing region info on filesystem at 1733897699180Initializing all the Stores at 1733897699181 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897699181Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897699181Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897699181Cleaning up temporary data from old regions at 1733897699204 (+23 ms)Running coprocessor post-open hooks at 1733897699207 (+3 ms)Region opened successfully at 1733897699208 (+1 ms) 2024-12-11T06:14:59,209 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., pid=35, masterSystemTime=1733897699176 2024-12-11T06:14:59,210 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,210 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=OPEN, openSeqNum=5, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:14:59,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-11T06:14:59,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 in 191 msec 2024-12-11T06:14:59,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-11T06:14:59,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, REOPEN/MOVE in 516 msec 2024-12-11T06:14:59,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-11T06:14:59,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-12-11T06:14:59,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0080 sec 2024-12-11T06:14:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T06:14:59,229 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:14:59,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc23ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,231 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a0725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,234 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce0ef3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,238 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc0c1f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,239 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d01cd97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,242 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfc72d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,243 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375bc779, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197ecaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f47fdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f10c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f01b8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99ec37e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adec721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16525882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4643b10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c2fc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c495221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35883207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6724e437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:14:59,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:14:59,261 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:14:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:14:59,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:14:59,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:14:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:14:59,264 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:14:59,265 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,264 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-11T06:14:59,266 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,266 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,267 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,267 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:14:59,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:14:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,268 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:14:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:14:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,269 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,276 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,277 DEBUG [Thread-612 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,278 DEBUG [Thread-612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,280 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:14:59,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:14:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:14:59,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:14:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:14:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:14:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:14:59,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121160abddca794c4bd992df5543543ee694_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897699289/Put/seqid=0 2024-12-11T06:14:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741945_1121 (size=14594) 2024-12-11T06:14:59,370 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:14:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:14:59,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897699353, max timestamp=1733897699402Num uniques 859; 2024-12-11T06:14:59,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:14:59,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897699353, max timestamp=1733897699402Num uniques 859; 2024-12-11T06:14:59,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:14:59,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121160abddca794c4bd992df5543543ee694_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121160abddca794c4bd992df5543543ee694_d0766cad7d1740597098828001c23b4a 2024-12-11T06:14:59,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897699353, max timestamp=1733897699402Num uniques 859; 2024-12-11T06:14:59,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:14:59,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a987aefb2d18461ca3870cfdb72523da, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:14:59,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:14:59,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:14:59,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a987aefb2d18461ca3870cfdb72523da is 175, key is test_row_0/A:col10/1733897699289/Put/seqid=0 2024-12-11T06:14:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897759450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897759447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741946_1122 (size=39549) 2024-12-11T06:14:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897759459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897759461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897759462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897759569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897759569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897759581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897759581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897759581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:14:59,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:14:59,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:14:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:14:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897759781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897759789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897759799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897759800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:14:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897759800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:14:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:14:59,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:14:59,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a987aefb2d18461ca3870cfdb72523da 2024-12-11T06:14:59,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:14:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:14:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:14:59,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:14:59,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/26c10d3171d540258c02d81fe6f6c61e is 50, key is test_row_0/B:col10/1733897699289/Put/seqid=0 2024-12-11T06:14:59,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741947_1123 (size=12001) 2024-12-11T06:14:59,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/26c10d3171d540258c02d81fe6f6c61e 2024-12-11T06:15:00,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/fbb567ebe4ea479eb047ab37a459226e is 50, key is test_row_0/C:col10/1733897699289/Put/seqid=0 2024-12-11T06:15:00,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741948_1124 (size=12001) 2024-12-11T06:15:00,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897760092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897760110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897760112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897760113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897760119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:15:00,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/fbb567ebe4ea479eb047ab37a459226e 2024-12-11T06:15:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a987aefb2d18461ca3870cfdb72523da as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da 2024-12-11T06:15:00,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da, entries=200, sequenceid=14, filesize=38.6 K 2024-12-11T06:15:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/26c10d3171d540258c02d81fe6f6c61e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e 2024-12-11T06:15:00,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-11T06:15:00,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/fbb567ebe4ea479eb047ab37a459226e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e 2024-12-11T06:15:00,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-11T06:15:00,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d0766cad7d1740597098828001c23b4a in 1298ms, sequenceid=14, compaction requested=false 2024-12-11T06:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-11T06:15:00,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:00,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:00,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b582b97313a640e6b628ccac0ebe5191_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897699456/Put/seqid=0 2024-12-11T06:15:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897760633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897760637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897760641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897760645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897760647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741949_1125 (size=14594) 2024-12-11T06:15:00,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b582b97313a640e6b628ccac0ebe5191_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b582b97313a640e6b628ccac0ebe5191_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:00,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/5578c019881c4843bade09978676fd75, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:00,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/5578c019881c4843bade09978676fd75 is 175, key is test_row_0/A:col10/1733897699456/Put/seqid=0 2024-12-11T06:15:00,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741950_1126 (size=39549) 2024-12-11T06:15:00,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/5578c019881c4843bade09978676fd75 2024-12-11T06:15:00,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/ea045af136c94b74b392a670fdf9430d is 50, key is test_row_0/B:col10/1733897699456/Put/seqid=0 2024-12-11T06:15:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897760749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897760759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897760759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897760759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897760768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:00,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741951_1127 (size=12001) 2024-12-11T06:15:00,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/ea045af136c94b74b392a670fdf9430d 2024-12-11T06:15:00,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/73dfad1429fb4cc5b5b44c4dd37ff71b is 50, key is test_row_0/C:col10/1733897699456/Put/seqid=0 2024-12-11T06:15:00,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741952_1128 (size=12001) 2024-12-11T06:15:00,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/73dfad1429fb4cc5b5b44c4dd37ff71b 2024-12-11T06:15:00,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:00,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:00,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/5578c019881c4843bade09978676fd75 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75 2024-12-11T06:15:00,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75, entries=200, sequenceid=51, filesize=38.6 K 2024-12-11T06:15:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/ea045af136c94b74b392a670fdf9430d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d 2024-12-11T06:15:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:15:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/73dfad1429fb4cc5b5b44c4dd37ff71b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b 2024-12-11T06:15:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for d0766cad7d1740597098828001c23b4a in 349ms, sequenceid=51, compaction requested=false 2024-12-11T06:15:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:15:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:00,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117ecc5cc10892497fbc1013450375800e_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741953_1129 (size=12154) 2024-12-11T06:15:01,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117ecc5cc10892497fbc1013450375800e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117ecc5cc10892497fbc1013450375800e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0638545d6de048e6a1cdc19bf211f01d, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0638545d6de048e6a1cdc19bf211f01d is 175, key is test_row_0/A:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741954_1130 (size=30951) 2024-12-11T06:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0638545d6de048e6a1cdc19bf211f01d 2024-12-11T06:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/1a15d27cf5ed4534a0c53429723d3b5e is 50, key is test_row_0/B:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741955_1131 (size=9657) 2024-12-11T06:15:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:01,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733897700990, max timestamp=1733897701094Num uniques 859; 2024-12-11T06:15:01,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-11T06:15:01,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733897700990, max timestamp=1733897701094Num uniques 859; 2024-12-11T06:15:01,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-11T06:15:01,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733897700990, max timestamp=1733897701094Num uniques 859; 2024-12-11T06:15:01,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-11T06:15:01,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897761105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897761105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897761107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897761107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897761109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897761218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897761219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897761219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897761219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897761219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,276 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:15:01,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897761432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897761441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897761441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:15:01,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/1a15d27cf5ed4534a0c53429723d3b5e 2024-12-11T06:15:01,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/395b51786ada4ed49787d8866d560409 is 50, key is test_row_0/C:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:01,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741956_1132 (size=9657) 2024-12-11T06:15:01,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897761740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897761749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897761749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897761750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897761750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:01,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:01,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:01,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:01,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/395b51786ada4ed49787d8866d560409 2024-12-11T06:15:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0638545d6de048e6a1cdc19bf211f01d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d 2024-12-11T06:15:02,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d, entries=150, sequenceid=61, filesize=30.2 K 2024-12-11T06:15:02,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/1a15d27cf5ed4534a0c53429723d3b5e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e 2024-12-11T06:15:02,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e, entries=100, sequenceid=61, filesize=9.4 K 2024-12-11T06:15:02,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/395b51786ada4ed49787d8866d560409 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409 2024-12-11T06:15:02,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409, entries=100, sequenceid=61, filesize=9.4 K 2024-12-11T06:15:02,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for d0766cad7d1740597098828001c23b4a in 1132ms, sequenceid=61, compaction requested=true 2024-12-11T06:15:02,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:02,120 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:02,120 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:02,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:02,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-11T06:15:02,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:02,127 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:02,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:02,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:02,129 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:02,129 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=107.5 K 2024-12-11T06:15:02,130 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:02,130 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d] 2024-12-11T06:15:02,130 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:02,130 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:02,131 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:02,131 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=32.9 K 2024-12-11T06:15:02,131 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a987aefb2d18461ca3870cfdb72523da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733897699289 2024-12-11T06:15:02,137 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c10d3171d540258c02d81fe6f6c61e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733897699289 2024-12-11T06:15:02,137 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5578c019881c4843bade09978676fd75, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897699437 2024-12-11T06:15:02,139 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ea045af136c94b74b392a670fdf9430d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897699437 2024-12-11T06:15:02,139 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0638545d6de048e6a1cdc19bf211f01d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897700640 2024-12-11T06:15:02,145 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a15d27cf5ed4534a0c53429723d3b5e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897700642 2024-12-11T06:15:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121123279b8ed6954c55b5b282a95599724f_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897701105/Put/seqid=0 2024-12-11T06:15:02,208 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:02,209 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/c8e25038fb044484a894260488c59e53 is 50, key is test_row_0/B:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741957_1133 (size=12154) 2024-12-11T06:15:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:02,229 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:02,246 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121123279b8ed6954c55b5b282a95599724f_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123279b8ed6954c55b5b282a95599724f_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:02,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a05a4aeecf554a1084431de2591ef206, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a05a4aeecf554a1084431de2591ef206 is 175, key is test_row_0/A:col10/1733897701105/Put/seqid=0 2024-12-11T06:15:02,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741958_1134 (size=12104) 2024-12-11T06:15:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:02,253 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211dc347b88d6464722aa7df6ffe6c05e7f_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:02,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:02,257 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211dc347b88d6464722aa7df6ffe6c05e7f_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:02,257 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211dc347b88d6464722aa7df6ffe6c05e7f_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:02,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741959_1135 (size=30955) 2024-12-11T06:15:02,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897762271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897762273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897762280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897762283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897762281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,308 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/c8e25038fb044484a894260488c59e53 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c8e25038fb044484a894260488c59e53 2024-12-11T06:15:02,316 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into c8e25038fb044484a894260488c59e53(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:02,316 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:02,316 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=13, startTime=1733897702120; duration=0sec 2024-12-11T06:15:02,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:02,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:02,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:02,320 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:02,320 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:02,320 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:02,320 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=32.9 K 2024-12-11T06:15:02,322 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb567ebe4ea479eb047ab37a459226e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733897699289 2024-12-11T06:15:02,322 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dfad1429fb4cc5b5b44c4dd37ff71b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897699437 2024-12-11T06:15:02,323 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 395b51786ada4ed49787d8866d560409, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897700642 2024-12-11T06:15:02,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741960_1136 (size=4469) 2024-12-11T06:15:02,344 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#116 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:02,345 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/fa4af8d945fa482c903e290dfffb4868 is 175, key is test_row_0/A:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:02,355 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:02,355 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/c1e7e29959d849f6935376e7578f1b78 is 50, key is test_row_0/C:col10/1733897700642/Put/seqid=0 2024-12-11T06:15:02,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741961_1137 (size=31165) 2024-12-11T06:15:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741962_1138 (size=12104) 2024-12-11T06:15:02,370 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/fa4af8d945fa482c903e290dfffb4868 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868 2024-12-11T06:15:02,374 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/c1e7e29959d849f6935376e7578f1b78 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1e7e29959d849f6935376e7578f1b78 2024-12-11T06:15:02,376 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into fa4af8d945fa482c903e290dfffb4868(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:02,376 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:02,376 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=13, startTime=1733897702119; duration=0sec 2024-12-11T06:15:02,376 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:02,376 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:02,380 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into c1e7e29959d849f6935376e7578f1b78(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:02,380 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:02,380 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=13, startTime=1733897702120; duration=0sec 2024-12-11T06:15:02,381 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:02,381 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897762388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897762388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897762398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897762398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897762399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897762598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897762598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897762609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897762609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897762609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,681 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a05a4aeecf554a1084431de2591ef206 2024-12-11T06:15:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/55bdd27d84414033bc897e1990c29286 is 50, key is test_row_0/B:col10/1733897701105/Put/seqid=0 2024-12-11T06:15:02,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741963_1139 (size=12001) 2024-12-11T06:15:02,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897762908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897762909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897762919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897762919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897762919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,094 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/55bdd27d84414033bc897e1990c29286 2024-12-11T06:15:03,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/81b0a628b5724a63b086b9d3fce3889f is 50, key is test_row_0/C:col10/1733897701105/Put/seqid=0 2024-12-11T06:15:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741964_1140 (size=12001) 2024-12-11T06:15:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897763419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897763419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897763429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897763429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897763429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:15:03,509 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/81b0a628b5724a63b086b9d3fce3889f 2024-12-11T06:15:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a05a4aeecf554a1084431de2591ef206 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206 2024-12-11T06:15:03,519 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206, entries=150, sequenceid=87, filesize=30.2 K 2024-12-11T06:15:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/55bdd27d84414033bc897e1990c29286 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286 2024-12-11T06:15:03,526 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286, entries=150, sequenceid=87, filesize=11.7 K 2024-12-11T06:15:03,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/81b0a628b5724a63b086b9d3fce3889f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f 2024-12-11T06:15:03,532 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f, entries=150, sequenceid=87, filesize=11.7 K 2024-12-11T06:15:03,534 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for d0766cad7d1740597098828001c23b4a in 1407ms, sequenceid=87, compaction requested=false 2024-12-11T06:15:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-11T06:15:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-11T06:15:03,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-11T06:15:03,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2660 sec 2024-12-11T06:15:03,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 4.2730 sec 2024-12-11T06:15:03,597 INFO [master/4518703b2c1d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-11T06:15:03,597 INFO [master/4518703b2c1d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-11T06:15:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:04,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:04,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211642455076ef64d1c9de65a0321d1fbd7_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:04,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741965_1141 (size=14594) 2024-12-11T06:15:04,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:04,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211642455076ef64d1c9de65a0321d1fbd7_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211642455076ef64d1c9de65a0321d1fbd7_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:04,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0087c5261e77415891255bc3a8ea6086, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:04,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0087c5261e77415891255bc3a8ea6086 is 175, key is test_row_0/A:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741966_1142 (size=39549) 2024-12-11T06:15:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897764464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897704462, max timestamp=1733897704486Num uniques 859; 2024-12-11T06:15:04,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:04,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897704462, max timestamp=1733897704486Num uniques 859; 2024-12-11T06:15:04,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:04,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897704462, max timestamp=1733897704486Num uniques 859; 2024-12-11T06:15:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897764514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897764515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897764515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897764516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897764599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897764628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897764628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897764628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897764628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897764808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897764839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897764839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897764839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897764839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:04,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0087c5261e77415891255bc3a8ea6086 2024-12-11T06:15:04,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/054bd91e58e44db897300817b92f6b04 is 50, key is test_row_0/B:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:04,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741967_1143 (size=12001) 2024-12-11T06:15:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897765118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897765149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897765149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897765149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897765149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,152 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/054bd91e58e44db897300817b92f6b04 2024-12-11T06:15:05,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/9f3c60e3064c454ead7e6a5a5c8c4a24 is 50, key is test_row_0/C:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:05,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741968_1144 (size=12001) 2024-12-11T06:15:05,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897765629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897765659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897765659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897765659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897765659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:05,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/9f3c60e3064c454ead7e6a5a5c8c4a24 2024-12-11T06:15:05,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0087c5261e77415891255bc3a8ea6086 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086 2024-12-11T06:15:05,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086, entries=200, sequenceid=101, filesize=38.6 K 2024-12-11T06:15:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/054bd91e58e44db897300817b92f6b04 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04 2024-12-11T06:15:05,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04, entries=150, sequenceid=101, filesize=11.7 K 2024-12-11T06:15:05,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/9f3c60e3064c454ead7e6a5a5c8c4a24 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24 2024-12-11T06:15:05,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24, entries=150, sequenceid=101, filesize=11.7 K 2024-12-11T06:15:05,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d0766cad7d1740597098828001c23b4a in 1280ms, sequenceid=101, compaction requested=true 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:05,712 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:05,712 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:05,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:05,713 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:05,713 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:05,713 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:05,713 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:05,713 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:05,713 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:05,713 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c8e25038fb044484a894260488c59e53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=35.3 K 2024-12-11T06:15:05,714 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=99.3 K 2024-12-11T06:15:05,714 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086] 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e25038fb044484a894260488c59e53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897699437 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4af8d945fa482c903e290dfffb4868, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897699437 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 55bdd27d84414033bc897e1990c29286, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897701103 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05a4aeecf554a1084431de2591ef206, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897701103 2024-12-11T06:15:05,714 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 054bd91e58e44db897300817b92f6b04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:05,715 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0087c5261e77415891255bc3a8ea6086, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:05,721 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:05,722 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:05,723 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/162b7b5575974e1ba3d9eed8af8670a8 is 50, key is test_row_0/B:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:05,724 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211b0edc590b4284ad28d6703fc73540e6f_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:05,725 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211b0edc590b4284ad28d6703fc73540e6f_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:05,726 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b0edc590b4284ad28d6703fc73540e6f_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741969_1145 (size=12207) 2024-12-11T06:15:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741970_1146 (size=4469) 2024-12-11T06:15:05,745 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/162b7b5575974e1ba3d9eed8af8670a8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/162b7b5575974e1ba3d9eed8af8670a8 2024-12-11T06:15:05,753 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into 162b7b5575974e1ba3d9eed8af8670a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:05,753 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:05,753 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=13, startTime=1733897705712; duration=0sec 2024-12-11T06:15:05,753 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:05,753 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:05,753 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:05,754 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:05,754 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:05,755 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:05,755 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1e7e29959d849f6935376e7578f1b78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=35.3 K 2024-12-11T06:15:05,755 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c1e7e29959d849f6935376e7578f1b78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733897699437 2024-12-11T06:15:05,756 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b0a628b5724a63b086b9d3fce3889f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1733897701103 2024-12-11T06:15:05,756 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3c60e3064c454ead7e6a5a5c8c4a24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:05,763 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:05,764 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/2dad40feabce4fc7bba66d486a313157 is 50, key is test_row_0/C:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:05,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741971_1147 (size=12207) 2024-12-11T06:15:06,139 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#124 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:06,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2bd279e8816746058d99f71442007203 is 175, key is test_row_0/A:col10/1733897702279/Put/seqid=0 2024-12-11T06:15:06,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741972_1148 (size=31161) 2024-12-11T06:15:06,150 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2bd279e8816746058d99f71442007203 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203 2024-12-11T06:15:06,174 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into 2bd279e8816746058d99f71442007203(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:06,174 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:06,174 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=13, startTime=1733897705712; duration=0sec 2024-12-11T06:15:06,174 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:06,174 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:06,178 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/2dad40feabce4fc7bba66d486a313157 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2dad40feabce4fc7bba66d486a313157 2024-12-11T06:15:06,184 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into 2dad40feabce4fc7bba66d486a313157(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:06,184 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:06,184 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=13, startTime=1733897705712; duration=0sec 2024-12-11T06:15:06,184 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:06,184 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:06,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:06,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:06,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211924d362a0ac0448d9dbb72623229ae3e_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897706640/Put/seqid=0 2024-12-11T06:15:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741973_1149 (size=14794) 2024-12-11T06:15:06,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897766668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897766678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,682 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897766678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897766678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897766679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,685 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,685 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,686 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897766778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:06,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897766988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:07,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:07,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211924d362a0ac0448d9dbb72623229ae3e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211924d362a0ac0448d9dbb72623229ae3e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:07,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0d7810e2620949508a665c94662d3118, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:07,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0d7810e2620949508a665c94662d3118 is 175, key is test_row_0/A:col10/1733897706640/Put/seqid=0 2024-12-11T06:15:07,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741974_1150 (size=39749) 2024-12-11T06:15:07,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897767299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:07,470 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0d7810e2620949508a665c94662d3118 2024-12-11T06:15:07,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/5c26dfdab8d74634b76784a931cb06b1 is 50, key is test_row_0/B:col10/1733897706640/Put/seqid=0 2024-12-11T06:15:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-11T06:15:07,479 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:07,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-11T06:15:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T06:15:07,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:07,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:07,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:07,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741975_1151 (size=12151) 2024-12-11T06:15:07,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/5c26dfdab8d74634b76784a931cb06b1 2024-12-11T06:15:07,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/43ea4a4a0992487885ea8e0dfcec849b is 50, key is test_row_0/C:col10/1733897706640/Put/seqid=0 2024-12-11T06:15:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741976_1152 (size=12151) 2024-12-11T06:15:07,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/43ea4a4a0992487885ea8e0dfcec849b 2024-12-11T06:15:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0d7810e2620949508a665c94662d3118 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118 2024-12-11T06:15:07,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118, entries=200, sequenceid=142, filesize=38.8 K 2024-12-11T06:15:07,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/5c26dfdab8d74634b76784a931cb06b1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1 2024-12-11T06:15:07,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1, entries=150, sequenceid=142, filesize=11.9 K 2024-12-11T06:15:07,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/43ea4a4a0992487885ea8e0dfcec849b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b 2024-12-11T06:15:07,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b, entries=150, sequenceid=142, filesize=11.9 K 2024-12-11T06:15:07,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for d0766cad7d1740597098828001c23b4a in 880ms, sequenceid=142, compaction requested=false 2024-12-11T06:15:07,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T06:15:07,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T06:15:07,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:07,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:07,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211ad9f875f9806409087ccce8c991bda57_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897706652/Put/seqid=0 2024-12-11T06:15:07,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741977_1153 (size=12304) 2024-12-11T06:15:07,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:07,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211ad9f875f9806409087ccce8c991bda57_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ad9f875f9806409087ccce8c991bda57_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/b07a7519c75644ac9c19b28081095b72, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:07,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/b07a7519c75644ac9c19b28081095b72 is 175, key is test_row_0/A:col10/1733897706652/Put/seqid=0 2024-12-11T06:15:07,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741978_1154 (size=31105) 2024-12-11T06:15:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T06:15:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:07,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:07,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897707871, max timestamp=1733897707888Num uniques 859; 2024-12-11T06:15:07,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:07,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897707871, max timestamp=1733897707888Num uniques 859; 2024-12-11T06:15:07,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:07,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897707871, max timestamp=1733897707888Num uniques 859; 2024-12-11T06:15:07,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:07,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897767957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:07,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,061 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/b07a7519c75644ac9c19b28081095b72 2024-12-11T06:15:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/84aaf5a0a71f45eba99803b072fcd52f is 50, key is test_row_0/B:col10/1733897706652/Put/seqid=0 2024-12-11T06:15:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897768070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741979_1155 (size=12151) 2024-12-11T06:15:08,074 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/84aaf5a0a71f45eba99803b072fcd52f 2024-12-11T06:15:08,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/3cede1024cd04631bf3d6a199fbeab8b is 50, key is test_row_0/C:col10/1733897706652/Put/seqid=0 2024-12-11T06:15:08,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741980_1156 (size=12151) 2024-12-11T06:15:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T06:15:08,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897768279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,503 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/3cede1024cd04631bf3d6a199fbeab8b 2024-12-11T06:15:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/b07a7519c75644ac9c19b28081095b72 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72 2024-12-11T06:15:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72, entries=150, sequenceid=151, filesize=30.4 K 2024-12-11T06:15:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/84aaf5a0a71f45eba99803b072fcd52f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f 2024-12-11T06:15:08,554 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f, entries=150, sequenceid=151, filesize=11.9 K 2024-12-11T06:15:08,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/3cede1024cd04631bf3d6a199fbeab8b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b 2024-12-11T06:15:08,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b, entries=150, sequenceid=151, filesize=11.9 K 2024-12-11T06:15:08,580 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d0766cad7d1740597098828001c23b4a in 945ms, sequenceid=151, compaction requested=true 2024-12-11T06:15:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-11T06:15:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-11T06:15:08,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-11T06:15:08,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-12-11T06:15:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:08,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:08,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.1200 sec 2024-12-11T06:15:08,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211490b9d9ca0c84f2fbf2335163d67e918_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T06:15:08,619 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:08,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-11T06:15:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:08,625 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:08,626 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:08,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:08,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741981_1157 (size=14794) 2024-12-11T06:15:08,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897768633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211490b9d9ca0c84f2fbf2335163d67e918_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211490b9d9ca0c84f2fbf2335163d67e918_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:08,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2d2d32254ca940e2bb35e7ddf6b6e1c7, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:08,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 is 175, key is test_row_0/A:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:08,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741982_1158 (size=39749) 2024-12-11T06:15:08,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 2024-12-11T06:15:08,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/152725ceedf945cca296baef3e6bfb9d is 50, key is test_row_0/B:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897768699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,703 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897768699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,706 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741983_1159 (size=12151) 2024-12-11T06:15:08,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/152725ceedf945cca296baef3e6bfb9d 2024-12-11T06:15:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897768708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897768708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,714 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,714 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:08,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d11e37bb260e4b79b990b6a8a2b4fd3d is 50, key is test_row_0/C:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897768740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:08,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741984_1160 (size=12151) 2024-12-11T06:15:08,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-11T06:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:08,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:08,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-11T06:15:08,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:08,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:08,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:08,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:08,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897768949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:09,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-11T06:15:09,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:09,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d11e37bb260e4b79b990b6a8a2b4fd3d 2024-12-11T06:15:09,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 2024-12-11T06:15:09,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7, entries=200, sequenceid=189, filesize=38.8 K 2024-12-11T06:15:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/152725ceedf945cca296baef3e6bfb9d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d 2024-12-11T06:15:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:09,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d, entries=150, sequenceid=189, filesize=11.9 K 2024-12-11T06:15:09,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d11e37bb260e4b79b990b6a8a2b4fd3d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d 2024-12-11T06:15:09,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897769269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d, entries=150, sequenceid=189, filesize=11.9 K 2024-12-11T06:15:09,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for d0766cad7d1740597098828001c23b4a in 683ms, sequenceid=189, compaction requested=true 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T06:15:09,283 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:09,284 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:09,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-11T06:15:09,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,286 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T06:15:09,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:09,289 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:09,289 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:09,290 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,290 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2dad40feabce4fc7bba66d486a313157, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=47.5 K 2024-12-11T06:15:09,290 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141764 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:09,290 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:09,290 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,290 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=138.4 K 2024-12-11T06:15:09,290 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,290 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7] 2024-12-11T06:15:09,291 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dad40feabce4fc7bba66d486a313157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:09,291 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd279e8816746058d99f71442007203, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:09,293 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ea4a4a0992487885ea8e0dfcec849b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733897704513 2024-12-11T06:15:09,297 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7810e2620949508a665c94662d3118, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733897704505 2024-12-11T06:15:09,301 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cede1024cd04631bf3d6a199fbeab8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733897706652 2024-12-11T06:15:09,301 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07a7519c75644ac9c19b28081095b72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733897706652 2024-12-11T06:15:09,301 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d11e37bb260e4b79b990b6a8a2b4fd3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707953 2024-12-11T06:15:09,302 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2d32254ca940e2bb35e7ddf6b6e1c7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707950 2024-12-11T06:15:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211c7c9bddae4cf461593f849e1b106a05d_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897708604/Put/seqid=0 2024-12-11T06:15:09,344 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:09,346 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/e6a51387cf094d24b9cad2f571f1d95d is 50, key is test_row_0/C:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:09,349 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:09,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741986_1162 (size=12493) 2024-12-11T06:15:09,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741985_1161 (size=12304) 2024-12-11T06:15:09,373 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211cae7d3f59584426fb73ea76a85b870da_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:09,376 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/e6a51387cf094d24b9cad2f571f1d95d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e6a51387cf094d24b9cad2f571f1d95d 2024-12-11T06:15:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:09,389 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211c7c9bddae4cf461593f849e1b106a05d_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c7c9bddae4cf461593f849e1b106a05d_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:09,389 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211cae7d3f59584426fb73ea76a85b870da_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:09,389 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211cae7d3f59584426fb73ea76a85b870da_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/9caa795ef1c345c7a38e17f479a5a879, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/9caa795ef1c345c7a38e17f479a5a879 is 175, key is test_row_0/A:col10/1733897708604/Put/seqid=0 2024-12-11T06:15:09,396 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into e6a51387cf094d24b9cad2f571f1d95d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:09,396 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:09,396 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=12, startTime=1733897709283; duration=0sec 2024-12-11T06:15:09,396 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:09,397 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:09,401 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:09,408 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:09,408 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:09,408 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:09,408 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/162b7b5575974e1ba3d9eed8af8670a8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=47.5 K 2024-12-11T06:15:09,411 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 162b7b5575974e1ba3d9eed8af8670a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733897702265 2024-12-11T06:15:09,417 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c26dfdab8d74634b76784a931cb06b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733897704513 2024-12-11T06:15:09,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741988_1164 (size=31105) 2024-12-11T06:15:09,422 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/9caa795ef1c345c7a38e17f479a5a879 2024-12-11T06:15:09,423 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 84aaf5a0a71f45eba99803b072fcd52f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733897706652 2024-12-11T06:15:09,427 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 152725ceedf945cca296baef3e6bfb9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707953 2024-12-11T06:15:09,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/e2fd2ebbf8094d23bbcc64503c1b8d53 is 50, key is test_row_0/B:col10/1733897708604/Put/seqid=0 2024-12-11T06:15:09,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741987_1163 (size=4469) 2024-12-11T06:15:09,455 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#137 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:09,456 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/87543780007d465ca3c8529450505f6a is 175, key is test_row_0/A:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:09,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741989_1165 (size=12151) 2024-12-11T06:15:09,476 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:09,477 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/7428ad04ddd440ca85bbaedb2af99842 is 50, key is test_row_0/B:col10/1733897707956/Put/seqid=0 2024-12-11T06:15:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741991_1167 (size=12493) 2024-12-11T06:15:09,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741990_1166 (size=31447) 2024-12-11T06:15:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:09,875 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/e2fd2ebbf8094d23bbcc64503c1b8d53 2024-12-11T06:15:09,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/41a9025c16034c79a51cc77648b39d30 is 50, key is test_row_0/C:col10/1733897708604/Put/seqid=0 2024-12-11T06:15:09,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741992_1168 (size=12151) 2024-12-11T06:15:09,939 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/7428ad04ddd440ca85bbaedb2af99842 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/7428ad04ddd440ca85bbaedb2af99842 2024-12-11T06:15:09,946 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into 7428ad04ddd440ca85bbaedb2af99842(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:09,946 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:09,946 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=12, startTime=1733897709283; duration=0sec 2024-12-11T06:15:09,946 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:09,946 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897769959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:09,963 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/87543780007d465ca3c8529450505f6a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a 2024-12-11T06:15:09,974 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into 87543780007d465ca3c8529450505f6a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:09,975 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:09,975 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=12, startTime=1733897709283; duration=0sec 2024-12-11T06:15:09,975 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:09,975 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:10,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897770070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897770280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:10,297 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/41a9025c16034c79a51cc77648b39d30 2024-12-11T06:15:10,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/9caa795ef1c345c7a38e17f479a5a879 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879 2024-12-11T06:15:10,317 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879, entries=150, sequenceid=197, filesize=30.4 K 2024-12-11T06:15:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/e2fd2ebbf8094d23bbcc64503c1b8d53 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53 2024-12-11T06:15:10,325 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T06:15:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/41a9025c16034c79a51cc77648b39d30 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30 2024-12-11T06:15:10,335 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T06:15:10,337 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=140.89 KB/144270 for d0766cad7d1740597098828001c23b4a in 1050ms, sequenceid=197, compaction requested=false 2024-12-11T06:15:10,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:10,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:10,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-11T06:15:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-11T06:15:10,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-11T06:15:10,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-12-11T06:15:10,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7250 sec 2024-12-11T06:15:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:10,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=147.60 KB heapSize=439.73 KB 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:10,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211789e93db56f747889eac3e499f2a8f44_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897770624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741993_1169 (size=14794) 2024-12-11T06:15:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897770741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T06:15:10,779 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:10,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-11T06:15:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:10,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:10,793 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:10,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:10,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:10,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897770959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:11,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:11,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211789e93db56f747889eac3e499f2a8f44_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211789e93db56f747889eac3e499f2a8f44_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:11,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a65d4e2584f64d59a9b125b451ec1dcf, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:11,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a65d4e2584f64d59a9b125b451ec1dcf is 175, key is test_row_0/A:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:11,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741994_1170 (size=39749) 2024-12-11T06:15:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897771269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:11,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:11,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,527 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a65d4e2584f64d59a9b125b451ec1dcf 2024-12-11T06:15:11,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/44afca5fef5a4b4aa9acec2326b62188 is 50, key is test_row_0/B:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:11,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741995_1171 (size=12151) 2024-12-11T06:15:11,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897771779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:11,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:11,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:11,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/44afca5fef5a4b4aa9acec2326b62188 2024-12-11T06:15:12,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/56338e0e87a94bc9b34ed3c98f7776a8 is 50, key is test_row_0/C:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:12,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741996_1172 (size=12151) 2024-12-11T06:15:12,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/56338e0e87a94bc9b34ed3c98f7776a8 2024-12-11T06:15:12,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/a65d4e2584f64d59a9b125b451ec1dcf as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf 2024-12-11T06:15:12,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:12,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf, entries=200, sequenceid=225, filesize=38.8 K 2024-12-11T06:15:12,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/44afca5fef5a4b4aa9acec2326b62188 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188 2024-12-11T06:15:12,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188, entries=150, sequenceid=225, filesize=11.9 K 2024-12-11T06:15:12,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/56338e0e87a94bc9b34ed3c98f7776a8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8 2024-12-11T06:15:12,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8, entries=150, sequenceid=225, filesize=11.9 K 2024-12-11T06:15:12,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~439.69 KB/450240, currentSize=26.84 KB/27480 for d0766cad7d1740597098828001c23b4a in 1503ms, sequenceid=225, compaction requested=true 2024-12-11T06:15:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:12,098 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:12,098 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:12,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:12,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:12,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:12,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:12,101 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:12,101 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:12,101 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:12,101 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:12,102 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,102 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,102 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/7428ad04ddd440ca85bbaedb2af99842, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=35.9 K 2024-12-11T06:15:12,102 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=99.9 K 2024-12-11T06:15:12,102 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,102 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf] 2024-12-11T06:15:12,102 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7428ad04ddd440ca85bbaedb2af99842, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707953 2024-12-11T06:15:12,104 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e2fd2ebbf8094d23bbcc64503c1b8d53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897708604 2024-12-11T06:15:12,104 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87543780007d465ca3c8529450505f6a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707953 2024-12-11T06:15:12,105 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9caa795ef1c345c7a38e17f479a5a879, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897708604 2024-12-11T06:15:12,106 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 44afca5fef5a4b4aa9acec2326b62188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709935 2024-12-11T06:15:12,107 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65d4e2584f64d59a9b125b451ec1dcf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709801 2024-12-11T06:15:12,122 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:12,124 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:12,124 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/33c98ad902954eddb441c00c4b720286 is 50, key is test_row_0/B:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:12,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121136e2025f04864fe1a58e6d3a47d8328f_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:12,131 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121136e2025f04864fe1a58e6d3a47d8328f_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:12,131 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121136e2025f04864fe1a58e6d3a47d8328f_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:12,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741997_1173 (size=12595) 2024-12-11T06:15:12,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741998_1174 (size=4469) 2024-12-11T06:15:12,165 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#144 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:12,165 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/04f96aec6d04498a80866ead64f2ec18 is 175, key is test_row_0/A:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:12,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741999_1175 (size=31549) 2024-12-11T06:15:12,196 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/04f96aec6d04498a80866ead64f2ec18 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18 2024-12-11T06:15:12,205 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into 04f96aec6d04498a80866ead64f2ec18(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:12,206 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:12,206 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=13, startTime=1733897712098; duration=0sec 2024-12-11T06:15:12,206 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:12,206 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:12,206 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:12,209 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:12,209 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:12,210 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,210 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e6a51387cf094d24b9cad2f571f1d95d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=35.9 K 2024-12-11T06:15:12,210 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6a51387cf094d24b9cad2f571f1d95d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733897707953 2024-12-11T06:15:12,211 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a9025c16034c79a51cc77648b39d30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733897708604 2024-12-11T06:15:12,211 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56338e0e87a94bc9b34ed3c98f7776a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709935 2024-12-11T06:15:12,227 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:12,227 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/79acfd8246544af593e8e9ba7400efe3 is 50, key is test_row_0/C:col10/1733897709955/Put/seqid=0 2024-12-11T06:15:12,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T06:15:12,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:12,233 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T06:15:12,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:12,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:12,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:12,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:12,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:12,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:12,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742000_1176 (size=12595) 2024-12-11T06:15:12,257 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/79acfd8246544af593e8e9ba7400efe3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/79acfd8246544af593e8e9ba7400efe3 2024-12-11T06:15:12,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d0e3727dc99545b2af1a9f05aa9c8bfd_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897710621/Put/seqid=0 2024-12-11T06:15:12,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742001_1177 (size=12304) 2024-12-11T06:15:12,285 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into 79acfd8246544af593e8e9ba7400efe3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:12,285 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:12,285 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=13, startTime=1733897712099; duration=0sec 2024-12-11T06:15:12,285 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:12,285 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:12,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:12,699 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d0e3727dc99545b2af1a9f05aa9c8bfd_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0e3727dc99545b2af1a9f05aa9c8bfd_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:12,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/c4998e7a21624eb99c3079bd4a30cfaa, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:12,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/c4998e7a21624eb99c3079bd4a30cfaa is 175, key is test_row_0/A:col10/1733897710621/Put/seqid=0 2024-12-11T06:15:12,721 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/33c98ad902954eddb441c00c4b720286 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/33c98ad902954eddb441c00c4b720286 2024-12-11T06:15:12,732 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into 33c98ad902954eddb441c00c4b720286(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:12,732 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:12,732 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=13, startTime=1733897712098; duration=0sec 2024-12-11T06:15:12,732 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:12,732 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742002_1178 (size=31105) 2024-12-11T06:15:12,736 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/c4998e7a21624eb99c3079bd4a30cfaa 2024-12-11T06:15:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:12,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:12,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/64f5b01f82d041b8828ac0dfabdb9bdc is 50, key is test_row_0/B:col10/1733897710621/Put/seqid=0 2024-12-11T06:15:12,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897712765, max timestamp=1733897712776Num uniques 859; 2024-12-11T06:15:12,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:12,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897712765, max timestamp=1733897712776Num uniques 859; 2024-12-11T06:15:12,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:12,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897712765, max timestamp=1733897712776Num uniques 859; 2024-12-11T06:15:12,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:12,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742003_1179 (size=12151) 2024-12-11T06:15:12,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12260, totalHeapSize=28112, min timestamp=1733897712807, max timestamp=1733897712825Num uniques 1068; 2024-12-11T06:15:12,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=12260, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-11T06:15:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897712807, max timestamp=1733897712825Num uniques 1018; 2024-12-11T06:15:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:15:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897712807, max timestamp=1733897712825Num uniques 1018; 2024-12-11T06:15:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:15:12,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897712853, max timestamp=1733897712895Num uniques 1127; 2024-12-11T06:15:12,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897712853, max timestamp=1733897712895Num uniques 1027; 2024-12-11T06:15:12,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-11T06:15:12,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-11T06:15:12,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897712853, max timestamp=1733897712895Num uniques 1027; 2024-12-11T06:15:12,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-11T06:15:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:12,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733897712924, max timestamp=1733897712935Num uniques 1086; 2024-12-11T06:15:12,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-11T06:15:12,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733897712924, max timestamp=1733897712935Num uniques 1086; 2024-12-11T06:15:12,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-11T06:15:12,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733897712924, max timestamp=1733897712935Num uniques 1086; 2024-12-11T06:15:12,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-11T06:15:13,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733897712967, max timestamp=1733897713020Num uniques 1095; 2024-12-11T06:15:13,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733897712967, max timestamp=1733897713020Num uniques 1095; 2024-12-11T06:15:13,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-11T06:15:13,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-11T06:15:13,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733897712967, max timestamp=1733897713020Num uniques 1095; 2024-12-11T06:15:13,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-12-11T06:15:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897773097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897773097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897773103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897773113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897773097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,194 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/64f5b01f82d041b8828ac0dfabdb9bdc 2024-12-11T06:15:13,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/e5771cc8e35a40bbb34bba39691fccf5 is 50, key is test_row_0/C:col10/1733897710621/Put/seqid=0 2024-12-11T06:15:13,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897773220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742004_1180 (size=12151) 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897773220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897773220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897773220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,228 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/e5771cc8e35a40bbb34bba39691fccf5 2024-12-11T06:15:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897773232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/c4998e7a21624eb99c3079bd4a30cfaa as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa 2024-12-11T06:15:13,278 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa, entries=150, sequenceid=233, filesize=30.4 K 2024-12-11T06:15:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/64f5b01f82d041b8828ac0dfabdb9bdc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc 2024-12-11T06:15:13,286 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc, entries=150, sequenceid=233, filesize=11.9 K 2024-12-11T06:15:13,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/e5771cc8e35a40bbb34bba39691fccf5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5 2024-12-11T06:15:13,301 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5, entries=150, sequenceid=233, filesize=11.9 K 2024-12-11T06:15:13,305 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for d0766cad7d1740597098828001c23b4a in 1071ms, sequenceid=233, compaction requested=false 2024-12-11T06:15:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-11T06:15:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-11T06:15:13,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-11T06:15:13,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5140 sec 2024-12-11T06:15:13,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5300 sec 2024-12-11T06:15:13,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897713055, max timestamp=1733897713431Num uniques 1204; 2024-12-11T06:15:13,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-12-11T06:15:13,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=120.76 KB heapSize=334.14 KB 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:13,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:15:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:13,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e34591bd3e85413a9a66a6bf58c3b77e_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742005_1181 (size=14994) 2024-12-11T06:15:13,463 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:13,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897773456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897773456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897773457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897773460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897773460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,470 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e34591bd3e85413a9a66a6bf58c3b77e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e34591bd3e85413a9a66a6bf58c3b77e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:13,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0a5b3dec31c14a858c6d4795dcba07b2, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:13,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0a5b3dec31c14a858c6d4795dcba07b2 is 175, key is test_row_0/A:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742006_1182 (size=39949) 2024-12-11T06:15:13,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0a5b3dec31c14a858c6d4795dcba07b2 2024-12-11T06:15:13,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/c0e20ac849ac40afbabaa06712d72509 is 50, key is test_row_0/B:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:13,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742007_1183 (size=12301) 2024-12-11T06:15:13,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/c0e20ac849ac40afbabaa06712d72509 2024-12-11T06:15:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897773568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/095d2085619f4a6490f5aeedd16ab624 is 50, key is test_row_0/C:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897773579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897773579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897773580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897773581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742008_1184 (size=12301) 2024-12-11T06:15:13,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897773782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897773788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897773788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897773788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897773799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/095d2085619f4a6490f5aeedd16ab624 2024-12-11T06:15:14,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/0a5b3dec31c14a858c6d4795dcba07b2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2 2024-12-11T06:15:14,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2, entries=200, sequenceid=342, filesize=39.0 K 2024-12-11T06:15:14,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/c0e20ac849ac40afbabaa06712d72509 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509 2024-12-11T06:15:14,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509, entries=150, sequenceid=342, filesize=12.0 K 2024-12-11T06:15:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/095d2085619f4a6490f5aeedd16ab624 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624 2024-12-11T06:15:14,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897774089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624, entries=150, sequenceid=342, filesize=12.0 K 2024-12-11T06:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~334.09 KB/342112, currentSize=60.38 KB/61830 for d0766cad7d1740597098828001c23b4a in 660ms, sequenceid=342, compaction requested=true 2024-12-11T06:15:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:14,095 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:14,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:14,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:14,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:15:14,097 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:14,097 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:14,097 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:14,098 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:14,098 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=100.2 K 2024-12-11T06:15:14,098 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:14,098 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2] 2024-12-11T06:15:14,100 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f96aec6d04498a80866ead64f2ec18, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709935 2024-12-11T06:15:14,102 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4998e7a21624eb99c3079bd4a30cfaa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733897710607 2024-12-11T06:15:14,102 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5b3dec31c14a858c6d4795dcba07b2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:14,105 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:14,105 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:14,105 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:14,105 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/33c98ad902954eddb441c00c4b720286, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=36.2 K 2024-12-11T06:15:14,106 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c98ad902954eddb441c00c4b720286, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709935 2024-12-11T06:15:14,107 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f5b01f82d041b8828ac0dfabdb9bdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733897710607 2024-12-11T06:15:14,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:15:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:14,108 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e20ac849ac40afbabaa06712d72509, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:14,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:14,131 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:14,135 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:14,135 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/82fdf6b566b24746bc8769868f3f1144 is 50, key is test_row_0/B:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:14,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412118a75e0261c4f4467b7de0b80460e62be_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897714106/Put/seqid=0 2024-12-11T06:15:14,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897774145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897774146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897774151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897774153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,161 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211b4844df3cd0849d5959ae86add9a6738_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,166 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211b4844df3cd0849d5959ae86add9a6738_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:14,166 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b4844df3cd0849d5959ae86add9a6738_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:14,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742010_1186 (size=12454) 2024-12-11T06:15:14,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742009_1185 (size=12847) 2024-12-11T06:15:14,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742011_1187 (size=4469) 2024-12-11T06:15:14,243 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#153 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:14,243 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/f44a8229386841acb3883189c9be18f5 is 175, key is test_row_0/A:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:14,247 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/82fdf6b566b24746bc8769868f3f1144 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/82fdf6b566b24746bc8769868f3f1144 2024-12-11T06:15:14,258 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into 82fdf6b566b24746bc8769868f3f1144(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:14,258 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:14,258 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=13, startTime=1733897714095; duration=0sec 2024-12-11T06:15:14,258 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:14,258 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:14,258 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:14,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:14,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:14,262 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:14,262 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/79acfd8246544af593e8e9ba7400efe3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=36.2 K 2024-12-11T06:15:14,266 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 79acfd8246544af593e8e9ba7400efe3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733897709935 2024-12-11T06:15:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897774261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,266 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e5771cc8e35a40bbb34bba39691fccf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733897710607 2024-12-11T06:15:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897774261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,267 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 095d2085619f4a6490f5aeedd16ab624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897774273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897774273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742012_1188 (size=31801) 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,292 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/f44a8229386841acb3883189c9be18f5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5 2024-12-11T06:15:14,298 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:14,299 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/1ee7d244e55a484ebdc7994f3465de93 is 50, key is test_row_0/C:col10/1733897713431/Put/seqid=0 2024-12-11T06:15:14,309 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into f44a8229386841acb3883189c9be18f5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:14,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:14,309 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=13, startTime=1733897714095; duration=0sec 2024-12-11T06:15:14,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:14,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742013_1189 (size=12847) 2024-12-11T06:15:14,357 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/1ee7d244e55a484ebdc7994f3465de93 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/1ee7d244e55a484ebdc7994f3465de93 2024-12-11T06:15:14,374 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into 1ee7d244e55a484ebdc7994f3465de93(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:14,374 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:14,374 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=13, startTime=1733897714095; duration=0sec 2024-12-11T06:15:14,374 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:14,374 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897774469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897774479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897774479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897774479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897774598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,600 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,618 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412118a75e0261c4f4467b7de0b80460e62be_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412118a75e0261c4f4467b7de0b80460e62be_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/99b5e13c23f542a3a40222f70328813d, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:14,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/99b5e13c23f542a3a40222f70328813d is 175, key is test_row_0/A:col10/1733897714106/Put/seqid=0 2024-12-11T06:15:14,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742014_1190 (size=31255) 2024-12-11T06:15:14,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/99b5e13c23f542a3a40222f70328813d 2024-12-11T06:15:14,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/77c4884b409b4a308ada1efd80cd9e69 is 50, key is test_row_0/B:col10/1733897714106/Put/seqid=0 2024-12-11T06:15:14,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742015_1191 (size=12301) 2024-12-11T06:15:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897774779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897774788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897774788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897774788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T06:15:14,939 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:14,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-11T06:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:14,944 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:14,945 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:14,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T06:15:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/77c4884b409b4a308ada1efd80cd9e69 2024-12-11T06:15:15,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/aab56a113e8d48f1bb94cd10f0d85d7d is 50, key is test_row_0/C:col10/1733897714106/Put/seqid=0 2024-12-11T06:15:15,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742016_1192 (size=12301) 2024-12-11T06:15:15,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T06:15:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:15,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897775289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897775300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897775300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:15,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897775301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:15,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T06:15:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/aab56a113e8d48f1bb94cd10f0d85d7d 2024-12-11T06:15:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:15,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T06:15:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:15,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/99b5e13c23f542a3a40222f70328813d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d 2024-12-11T06:15:15,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d, entries=150, sequenceid=356, filesize=30.5 K 2024-12-11T06:15:15,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/77c4884b409b4a308ada1efd80cd9e69 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69 2024-12-11T06:15:15,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897775609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:15,612 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:15,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69, entries=150, sequenceid=356, filesize=12.0 K 2024-12-11T06:15:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/aab56a113e8d48f1bb94cd10f0d85d7d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d 2024-12-11T06:15:15,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d, entries=150, sequenceid=356, filesize=12.0 K 2024-12-11T06:15:15,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d0766cad7d1740597098828001c23b4a in 1523ms, sequenceid=356, compaction requested=false 2024-12-11T06:15:15,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:15,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:15,727 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:15,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:15,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121193bf9e0211c549488dee22d9191bd68e_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897714150/Put/seqid=0 2024-12-11T06:15:15,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742017_1193 (size=12454) 2024-12-11T06:15:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:16,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:16,210 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121193bf9e0211c549488dee22d9191bd68e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121193bf9e0211c549488dee22d9191bd68e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:16,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7542007582914ff3bf7933c99e2ba279, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7542007582914ff3bf7933c99e2ba279 is 175, key is test_row_0/A:col10/1733897714150/Put/seqid=0 2024-12-11T06:15:16,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742018_1194 (size=31255) 2024-12-11T06:15:16,231 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7542007582914ff3bf7933c99e2ba279 2024-12-11T06:15:16,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/300901d83dc644a89fcff2deb989a844 is 50, key is test_row_0/B:col10/1733897714150/Put/seqid=0 2024-12-11T06:15:16,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742019_1195 (size=12301) 2024-12-11T06:15:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:16,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897776338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897776339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897776346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897776346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897776449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897776450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897776458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897776459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T06:15:16,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897776659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897776660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,668 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/300901d83dc644a89fcff2deb989a844 2024-12-11T06:15:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897776692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897776692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/2a8289ed4e384c3c893b2c72eba7e9ba is 50, key is test_row_0/C:col10/1733897714150/Put/seqid=0 2024-12-11T06:15:16,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742020_1196 (size=12301) 2024-12-11T06:15:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897776969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:16,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897776978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897777009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897777013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:17,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/2a8289ed4e384c3c893b2c72eba7e9ba 2024-12-11T06:15:17,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7542007582914ff3bf7933c99e2ba279 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279 2024-12-11T06:15:17,157 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279, entries=150, sequenceid=377, filesize=30.5 K 2024-12-11T06:15:17,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/300901d83dc644a89fcff2deb989a844 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844 2024-12-11T06:15:17,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844, entries=150, sequenceid=377, filesize=12.0 K 2024-12-11T06:15:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/2a8289ed4e384c3c893b2c72eba7e9ba as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba 2024-12-11T06:15:17,182 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba, entries=150, sequenceid=377, filesize=12.0 K 2024-12-11T06:15:17,185 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d0766cad7d1740597098828001c23b4a in 1457ms, sequenceid=377, compaction requested=true 2024-12-11T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:17,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-11T06:15:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-11T06:15:17,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-11T06:15:17,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2470 sec 2024-12-11T06:15:17,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2650 sec 2024-12-11T06:15:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:17,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:17,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:17,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d6326844f4ac4cc396a21afee364c769_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:17,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742021_1197 (size=14994) 2024-12-11T06:15:17,543 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897777531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897777534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897777534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897777544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d6326844f4ac4cc396a21afee364c769_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d6326844f4ac4cc396a21afee364c769_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:17,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/3f5cf67cd8a040fb89ebb04822e2bf34, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:17,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/3f5cf67cd8a040fb89ebb04822e2bf34 is 175, key is test_row_0/A:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:17,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742022_1198 (size=39949) 2024-12-11T06:15:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54856 deadline: 1733897777639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,642 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in d0766cad7d1740597098828001c23b4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897777648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897777648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897777648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897777858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897777858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897777858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/3f5cf67cd8a040fb89ebb04822e2bf34 2024-12-11T06:15:18,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/908d1435f86245e28262da4c22c80bc6 is 50, key is test_row_0/B:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:18,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742023_1199 (size=12301) 2024-12-11T06:15:18,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/908d1435f86245e28262da4c22c80bc6 2024-12-11T06:15:18,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d61b0e0cbeb6494d836cef0a24e3b5dc is 50, key is test_row_0/C:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:18,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742024_1200 (size=12301) 2024-12-11T06:15:18,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d61b0e0cbeb6494d836cef0a24e3b5dc 2024-12-11T06:15:18,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/3f5cf67cd8a040fb89ebb04822e2bf34 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34 2024-12-11T06:15:18,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34, entries=200, sequenceid=392, filesize=39.0 K 2024-12-11T06:15:18,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/908d1435f86245e28262da4c22c80bc6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6 2024-12-11T06:15:18,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6, entries=150, sequenceid=392, filesize=12.0 K 2024-12-11T06:15:18,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/d61b0e0cbeb6494d836cef0a24e3b5dc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc 2024-12-11T06:15:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897778169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897778169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897778169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc, entries=150, sequenceid=392, filesize=12.0 K 2024-12-11T06:15:18,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for d0766cad7d1740597098828001c23b4a in 700ms, sequenceid=392, compaction requested=true 2024-12-11T06:15:18,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:18,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:18,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:18,181 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:18,182 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0766cad7d1740597098828001c23b4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:18,190 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:18,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:18,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/B is initiating minor compaction (all files) 2024-12-11T06:15:18,190 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/A is initiating minor compaction (all files) 2024-12-11T06:15:18,191 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/B in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:18,191 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/82fdf6b566b24746bc8769868f3f1144, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=48.6 K 2024-12-11T06:15:18,191 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/A in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:18,191 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=131.1 K 2024-12-11T06:15:18,191 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:18,191 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34] 2024-12-11T06:15:18,192 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44a8229386841acb3883189c9be18f5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:18,192 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fdf6b566b24746bc8769868f3f1144, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:18,197 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c4884b409b4a308ada1efd80cd9e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733897713454 2024-12-11T06:15:18,197 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b5e13c23f542a3a40222f70328813d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733897713454 2024-12-11T06:15:18,201 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 300901d83dc644a89fcff2deb989a844, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733897714143 2024-12-11T06:15:18,202 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7542007582914ff3bf7933c99e2ba279, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733897714143 2024-12-11T06:15:18,202 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 908d1435f86245e28262da4c22c80bc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733897716320 2024-12-11T06:15:18,203 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5cf67cd8a040fb89ebb04822e2bf34, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733897716320 2024-12-11T06:15:18,239 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:18,242 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:18,243 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/d113ebe8197d4825bd427dd23e9abfdc is 50, key is test_row_0/B:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:18,248 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412119e7bd419af3a4cd9b3c202046b8b95d8_d0766cad7d1740597098828001c23b4a store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:18,251 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412119e7bd419af3a4cd9b3c202046b8b95d8_d0766cad7d1740597098828001c23b4a, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:18,252 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119e7bd419af3a4cd9b3c202046b8b95d8_d0766cad7d1740597098828001c23b4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742025_1201 (size=12983) 2024-12-11T06:15:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742026_1202 (size=4469) 2024-12-11T06:15:18,283 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#A#compaction#165 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:18,284 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/04e270e468b545aca9521eef6a07b5c1 is 175, key is test_row_0/A:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:18,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742027_1203 (size=31937) 2024-12-11T06:15:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:18,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:18,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:18,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115bcf230ae6d543a0a64dc11e7338a6e5_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897718560/Put/seqid=0 2024-12-11T06:15:18,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742028_1204 (size=14994) 2024-12-11T06:15:18,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:18,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897778627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115bcf230ae6d543a0a64dc11e7338a6e5_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115bcf230ae6d543a0a64dc11e7338a6e5_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:18,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/72dd01f9de18496b95ad71c8b4869eae, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:18,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/72dd01f9de18496b95ad71c8b4869eae is 175, key is test_row_0/A:col10/1733897718560/Put/seqid=0 2024-12-11T06:15:18,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742029_1205 (size=39949) 2024-12-11T06:15:18,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/72dd01f9de18496b95ad71c8b4869eae 2024-12-11T06:15:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54838 deadline: 1733897778678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54842 deadline: 1733897778681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/d041e69251194710bfd5a45984659b99 is 50, key is test_row_0/B:col10/1733897718560/Put/seqid=0 2024-12-11T06:15:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54906 deadline: 1733897778690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,699 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/d113ebe8197d4825bd427dd23e9abfdc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d113ebe8197d4825bd427dd23e9abfdc 2024-12-11T06:15:18,717 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/04e270e468b545aca9521eef6a07b5c1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04e270e468b545aca9521eef6a07b5c1 2024-12-11T06:15:18,718 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/B of d0766cad7d1740597098828001c23b4a into d113ebe8197d4825bd427dd23e9abfdc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:18,718 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:18,718 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/B, priority=12, startTime=1733897718181; duration=0sec 2024-12-11T06:15:18,718 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:18,718 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:B 2024-12-11T06:15:18,718 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:18,721 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:18,721 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): d0766cad7d1740597098828001c23b4a/C is initiating minor compaction (all files) 2024-12-11T06:15:18,721 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0766cad7d1740597098828001c23b4a/C in TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:18,721 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/1ee7d244e55a484ebdc7994f3465de93, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp, totalSize=48.6 K 2024-12-11T06:15:18,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742030_1206 (size=12301) 2024-12-11T06:15:18,723 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ee7d244e55a484ebdc7994f3465de93, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733897713055 2024-12-11T06:15:18,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/d041e69251194710bfd5a45984659b99 2024-12-11T06:15:18,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting aab56a113e8d48f1bb94cd10f0d85d7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733897713454 2024-12-11T06:15:18,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8289ed4e384c3c893b2c72eba7e9ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733897714143 2024-12-11T06:15:18,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d61b0e0cbeb6494d836cef0a24e3b5dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733897716320 2024-12-11T06:15:18,729 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/A of d0766cad7d1740597098828001c23b4a into 04e270e468b545aca9521eef6a07b5c1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:18,730 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:18,730 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/A, priority=12, startTime=1733897718181; duration=0sec 2024-12-11T06:15:18,731 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:18,731 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:A 2024-12-11T06:15:18,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/c1b3afd0608a46bca461403441f7b6f5 is 50, key is test_row_0/C:col10/1733897718560/Put/seqid=0 2024-12-11T06:15:18,743 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0766cad7d1740597098828001c23b4a#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:18,743 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/a0f4e2496be24ed0899b8bc90f76acf5 is 50, key is test_row_0/C:col10/1733897716320/Put/seqid=0 2024-12-11T06:15:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897778738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:18,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742031_1207 (size=12301) 2024-12-11T06:15:18,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742032_1208 (size=12983) 2024-12-11T06:15:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897778948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T06:15:19,109 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:19,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-11T06:15:19,111 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:19,111 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:19,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:19,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/c1b3afd0608a46bca461403441f7b6f5 2024-12-11T06:15:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:19,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/72dd01f9de18496b95ad71c8b4869eae as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/72dd01f9de18496b95ad71c8b4869eae 2024-12-11T06:15:19,227 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/a0f4e2496be24ed0899b8bc90f76acf5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/a0f4e2496be24ed0899b8bc90f76acf5 2024-12-11T06:15:19,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/72dd01f9de18496b95ad71c8b4869eae, entries=200, sequenceid=412, filesize=39.0 K 2024-12-11T06:15:19,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/d041e69251194710bfd5a45984659b99 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d041e69251194710bfd5a45984659b99 2024-12-11T06:15:19,256 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0766cad7d1740597098828001c23b4a/C of d0766cad7d1740597098828001c23b4a into a0f4e2496be24ed0899b8bc90f76acf5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:19,256 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:19,256 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., storeName=d0766cad7d1740597098828001c23b4a/C, priority=12, startTime=1733897718182; duration=0sec 2024-12-11T06:15:19,256 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:19,256 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0766cad7d1740597098828001c23b4a:C 2024-12-11T06:15:19,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-11T06:15:19,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:19,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:19,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:19,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54880 deadline: 1733897779262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:19,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:19,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:19,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:19,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d041e69251194710bfd5a45984659b99, entries=150, sequenceid=412, filesize=12.0 K 2024-12-11T06:15:19,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/c1b3afd0608a46bca461403441f7b6f5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1b3afd0608a46bca461403441f7b6f5 2024-12-11T06:15:19,282 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-12-11T06:15:19,282 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,282 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,282 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,284 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-12-11T06:15:19,284 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,284 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,284 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,286 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-12-11T06:15:19,286 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,286 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,286 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1b3afd0608a46bca461403441f7b6f5, entries=150, sequenceid=412, filesize=12.0 K 2024-12-11T06:15:19,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d0766cad7d1740597098828001c23b4a in 728ms, sequenceid=412, compaction requested=false 2024-12-11T06:15:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:19,292 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-12-11T06:15:19,292 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,292 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,292 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-11T06:15:19,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:19,420 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:19,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110b8364cf31b34bca9ad8f5ed239d88b8_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897718624/Put/seqid=0 2024-12-11T06:15:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742033_1209 (size=12454) 2024-12-11T06:15:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. as already flushing 2024-12-11T06:15:19,700 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-11T06:15:19,700 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-11T06:15:19,700 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,700 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,700 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,700 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,701 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,701 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,710 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-11T06:15:19,710 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,710 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,711 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:19,770 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-12-11T06:15:19,771 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:19,771 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,771 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:19,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:19,843 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110b8364cf31b34bca9ad8f5ed239d88b8_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110b8364cf31b34bca9ad8f5ed239d88b8_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7a8bea4286d74f518a085a61b900bf3a, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7a8bea4286d74f518a085a61b900bf3a is 175, key is test_row_0/A:col10/1733897718624/Put/seqid=0 2024-12-11T06:15:19,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742034_1210 (size=31255) 2024-12-11T06:15:19,850 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7a8bea4286d74f518a085a61b900bf3a 2024-12-11T06:15:19,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/3e97fb44f4224613bbd6b2a15debd00f is 50, key is test_row_0/B:col10/1733897718624/Put/seqid=0 2024-12-11T06:15:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742035_1211 (size=12301) 2024-12-11T06:15:19,867 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/3e97fb44f4224613bbd6b2a15debd00f 2024-12-11T06:15:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/bdfcb0f733fe411cb409f19585a0796c is 50, key is test_row_0/C:col10/1733897718624/Put/seqid=0 2024-12-11T06:15:19,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742036_1212 (size=12301) 2024-12-11T06:15:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:20,284 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/bdfcb0f733fe411cb409f19585a0796c 2024-12-11T06:15:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/7a8bea4286d74f518a085a61b900bf3a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7a8bea4286d74f518a085a61b900bf3a 2024-12-11T06:15:20,297 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7a8bea4286d74f518a085a61b900bf3a, entries=150, sequenceid=427, filesize=30.5 K 2024-12-11T06:15:20,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/3e97fb44f4224613bbd6b2a15debd00f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/3e97fb44f4224613bbd6b2a15debd00f 2024-12-11T06:15:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/3e97fb44f4224613bbd6b2a15debd00f, entries=150, sequenceid=427, filesize=12.0 K 2024-12-11T06:15:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/bdfcb0f733fe411cb409f19585a0796c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/bdfcb0f733fe411cb409f19585a0796c 2024-12-11T06:15:20,307 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/bdfcb0f733fe411cb409f19585a0796c, entries=150, sequenceid=427, filesize=12.0 K 2024-12-11T06:15:20,307 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for d0766cad7d1740597098828001c23b4a in 887ms, sequenceid=427, compaction requested=true 2024-12-11T06:15:20,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d0766cad7d1740597098828001c23b4a: 2024-12-11T06:15:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-11T06:15:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-11T06:15:20,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-11T06:15:20,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-12-11T06:15:20,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2020 sec 2024-12-11T06:15:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T06:15:21,259 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:21,660 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-12-11T06:15:21,660 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:21,661 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:21,661 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:21,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3079 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2980 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1376 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4127 rows 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1359 2024-12-11T06:15:21,661 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4076 rows 2024-12-11T06:15:21,661 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:15:21,661 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:15:21,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:21,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:21,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:21,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:15:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:21,666 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897721665"}]},"ts":"1733897721665"} 2024-12-11T06:15:21,667 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:15:21,667 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:15:21,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:15:21,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, UNASSIGN}] 2024-12-11T06:15:21,670 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, UNASSIGN 2024-12-11T06:15:21,671 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:21,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, UNASSIGN because future has completed 2024-12-11T06:15:21,673 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:15:21,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:21,825 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:21,825 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:15:21,825 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing d0766cad7d1740597098828001c23b4a, disabling compactions & flushes 2024-12-11T06:15:21,825 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:21,825 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:21,825 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. after waiting 0 ms 2024-12-11T06:15:21,825 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:21,825 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing d0766cad7d1740597098828001c23b4a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=A 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=B 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0766cad7d1740597098828001c23b4a, store=C 2024-12-11T06:15:21,826 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:21,831 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121191add167004045a4b71374f56fe5f284_d0766cad7d1740597098828001c23b4a is 50, key is test_row_0/A:col10/1733897719769/Put/seqid=0 2024-12-11T06:15:21,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742037_1213 (size=9914) 2024-12-11T06:15:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:22,235 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:22,239 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121191add167004045a4b71374f56fe5f284_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121191add167004045a4b71374f56fe5f284_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:22,244 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/6e9e5edfe9ab47898a952e32919035e6, store: [table=TestAcidGuarantees family=A region=d0766cad7d1740597098828001c23b4a] 2024-12-11T06:15:22,244 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/6e9e5edfe9ab47898a952e32919035e6 is 175, key is test_row_0/A:col10/1733897719769/Put/seqid=0 2024-12-11T06:15:22,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742038_1214 (size=22561) 2024-12-11T06:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:22,648 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/6e9e5edfe9ab47898a952e32919035e6 2024-12-11T06:15:22,655 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/52538936900a4cd6971f98ae109ceaba is 50, key is test_row_0/B:col10/1733897719769/Put/seqid=0 2024-12-11T06:15:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742039_1215 (size=9857) 2024-12-11T06:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:23,068 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/52538936900a4cd6971f98ae109ceaba 2024-12-11T06:15:23,075 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/47bab07613364769b9ce9ffd21f88b1e is 50, key is test_row_0/C:col10/1733897719769/Put/seqid=0 2024-12-11T06:15:23,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742040_1216 (size=9857) 2024-12-11T06:15:23,481 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/47bab07613364769b9ce9ffd21f88b1e 2024-12-11T06:15:23,486 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/A/6e9e5edfe9ab47898a952e32919035e6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/6e9e5edfe9ab47898a952e32919035e6 2024-12-11T06:15:23,490 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/6e9e5edfe9ab47898a952e32919035e6, entries=100, sequenceid=435, filesize=22.0 K 2024-12-11T06:15:23,491 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/B/52538936900a4cd6971f98ae109ceaba as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/52538936900a4cd6971f98ae109ceaba 2024-12-11T06:15:23,494 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/52538936900a4cd6971f98ae109ceaba, entries=100, sequenceid=435, filesize=9.6 K 2024-12-11T06:15:23,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/.tmp/C/47bab07613364769b9ce9ffd21f88b1e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/47bab07613364769b9ce9ffd21f88b1e 2024-12-11T06:15:23,499 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/47bab07613364769b9ce9ffd21f88b1e, entries=100, sequenceid=435, filesize=9.6 K 2024-12-11T06:15:23,500 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d0766cad7d1740597098828001c23b4a in 1675ms, sequenceid=435, compaction requested=true 2024-12-11T06:15:23,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34] to archive 2024-12-11T06:15:23,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:23,505 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a05a4aeecf554a1084431de2591ef206 2024-12-11T06:15:23,505 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a987aefb2d18461ca3870cfdb72523da 2024-12-11T06:15:23,505 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/5578c019881c4843bade09978676fd75 2024-12-11T06:15:23,505 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/fa4af8d945fa482c903e290dfffb4868 2024-12-11T06:15:23,505 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0638545d6de048e6a1cdc19bf211f01d 2024-12-11T06:15:23,506 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0d7810e2620949508a665c94662d3118 2024-12-11T06:15:23,506 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2bd279e8816746058d99f71442007203 2024-12-11T06:15:23,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0087c5261e77415891255bc3a8ea6086 2024-12-11T06:15:23,509 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/b07a7519c75644ac9c19b28081095b72 2024-12-11T06:15:23,509 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/9caa795ef1c345c7a38e17f479a5a879 2024-12-11T06:15:23,509 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04f96aec6d04498a80866ead64f2ec18 2024-12-11T06:15:23,510 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/2d2d32254ca940e2bb35e7ddf6b6e1c7 2024-12-11T06:15:23,510 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/87543780007d465ca3c8529450505f6a 2024-12-11T06:15:23,510 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/a65d4e2584f64d59a9b125b451ec1dcf 2024-12-11T06:15:23,511 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/c4998e7a21624eb99c3079bd4a30cfaa 2024-12-11T06:15:23,511 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/0a5b3dec31c14a858c6d4795dcba07b2 2024-12-11T06:15:23,511 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/3f5cf67cd8a040fb89ebb04822e2bf34 2024-12-11T06:15:23,511 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/f44a8229386841acb3883189c9be18f5 2024-12-11T06:15:23,511 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/99b5e13c23f542a3a40222f70328813d 2024-12-11T06:15:23,512 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7542007582914ff3bf7933c99e2ba279 2024-12-11T06:15:23,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c8e25038fb044484a894260488c59e53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/162b7b5575974e1ba3d9eed8af8670a8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/7428ad04ddd440ca85bbaedb2af99842, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/33c98ad902954eddb441c00c4b720286, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/82fdf6b566b24746bc8769868f3f1144, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6] to archive 2024-12-11T06:15:23,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:23,518 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/ea045af136c94b74b392a670fdf9430d 2024-12-11T06:15:23,518 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/26c10d3171d540258c02d81fe6f6c61e 2024-12-11T06:15:23,518 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c8e25038fb044484a894260488c59e53 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c8e25038fb044484a894260488c59e53 2024-12-11T06:15:23,518 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/5c26dfdab8d74634b76784a931cb06b1 2024-12-11T06:15:23,519 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/1a15d27cf5ed4534a0c53429723d3b5e 2024-12-11T06:15:23,519 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/054bd91e58e44db897300817b92f6b04 2024-12-11T06:15:23,519 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/55bdd27d84414033bc897e1990c29286 2024-12-11T06:15:23,519 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/162b7b5575974e1ba3d9eed8af8670a8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/162b7b5575974e1ba3d9eed8af8670a8 2024-12-11T06:15:23,519 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/7428ad04ddd440ca85bbaedb2af99842 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/7428ad04ddd440ca85bbaedb2af99842 2024-12-11T06:15:23,520 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/84aaf5a0a71f45eba99803b072fcd52f 2024-12-11T06:15:23,520 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/33c98ad902954eddb441c00c4b720286 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/33c98ad902954eddb441c00c4b720286 2024-12-11T06:15:23,520 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/152725ceedf945cca296baef3e6bfb9d 2024-12-11T06:15:23,521 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/e2fd2ebbf8094d23bbcc64503c1b8d53 2024-12-11T06:15:23,521 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/44afca5fef5a4b4aa9acec2326b62188 2024-12-11T06:15:23,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/64f5b01f82d041b8828ac0dfabdb9bdc 2024-12-11T06:15:23,522 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/82fdf6b566b24746bc8769868f3f1144 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/82fdf6b566b24746bc8769868f3f1144 2024-12-11T06:15:23,523 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/300901d83dc644a89fcff2deb989a844 2024-12-11T06:15:23,523 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/77c4884b409b4a308ada1efd80cd9e69 2024-12-11T06:15:23,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/c0e20ac849ac40afbabaa06712d72509 2024-12-11T06:15:23,523 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/908d1435f86245e28262da4c22c80bc6 2024-12-11T06:15:23,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1e7e29959d849f6935376e7578f1b78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2dad40feabce4fc7bba66d486a313157, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e6a51387cf094d24b9cad2f571f1d95d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/79acfd8246544af593e8e9ba7400efe3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/1ee7d244e55a484ebdc7994f3465de93, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc] to archive 2024-12-11T06:15:23,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:23,528 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1e7e29959d849f6935376e7578f1b78 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1e7e29959d849f6935376e7578f1b78 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/fbb567ebe4ea479eb047ab37a459226e 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/81b0a628b5724a63b086b9d3fce3889f 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/9f3c60e3064c454ead7e6a5a5c8c4a24 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/73dfad1429fb4cc5b5b44c4dd37ff71b 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/43ea4a4a0992487885ea8e0dfcec849b 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/395b51786ada4ed49787d8866d560409 2024-12-11T06:15:23,529 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2dad40feabce4fc7bba66d486a313157 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2dad40feabce4fc7bba66d486a313157 2024-12-11T06:15:23,531 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/3cede1024cd04631bf3d6a199fbeab8b 2024-12-11T06:15:23,531 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e6a51387cf094d24b9cad2f571f1d95d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e6a51387cf094d24b9cad2f571f1d95d 2024-12-11T06:15:23,531 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d11e37bb260e4b79b990b6a8a2b4fd3d 2024-12-11T06:15:23,531 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/56338e0e87a94bc9b34ed3c98f7776a8 2024-12-11T06:15:23,531 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/79acfd8246544af593e8e9ba7400efe3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/79acfd8246544af593e8e9ba7400efe3 2024-12-11T06:15:23,532 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/e5771cc8e35a40bbb34bba39691fccf5 2024-12-11T06:15:23,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/41a9025c16034c79a51cc77648b39d30 2024-12-11T06:15:23,532 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/1ee7d244e55a484ebdc7994f3465de93 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/1ee7d244e55a484ebdc7994f3465de93 2024-12-11T06:15:23,533 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/095d2085619f4a6490f5aeedd16ab624 2024-12-11T06:15:23,533 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/aab56a113e8d48f1bb94cd10f0d85d7d 2024-12-11T06:15:23,533 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/2a8289ed4e384c3c893b2c72eba7e9ba 2024-12-11T06:15:23,533 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/d61b0e0cbeb6494d836cef0a24e3b5dc 2024-12-11T06:15:23,537 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits/438.seqid, newMaxSeqId=438, maxSeqId=4 2024-12-11T06:15:23,537 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. 2024-12-11T06:15:23,537 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for d0766cad7d1740597098828001c23b4a: Waiting for close lock at 1733897721825Running coprocessor pre-close hooks at 1733897721825Disabling compacts and flushes for region at 1733897721825Disabling writes for close at 1733897721825Obtaining lock to block concurrent updates at 1733897721825Preparing flush snapshotting stores in d0766cad7d1740597098828001c23b4a at 1733897721825Finished memstore snapshotting TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733897721826 (+1 ms)Flushing stores of TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a. at 1733897721826Flushing d0766cad7d1740597098828001c23b4a/A: creating writer at 1733897721826Flushing mob file d0766cad7d1740597098828001c23b4a/A: appending metadata at 1733897721831 (+5 ms)Flushing mob file d0766cad7d1740597098828001c23b4a/A: closing flushed file at 1733897721831Flushing d0766cad7d1740597098828001c23b4a/A: appending metadata at 1733897722244 (+413 ms)Flushing d0766cad7d1740597098828001c23b4a/A: closing flushed file at 1733897722244Flushing d0766cad7d1740597098828001c23b4a/B: creating writer at 1733897722652 (+408 ms)Flushing d0766cad7d1740597098828001c23b4a/B: appending metadata at 1733897722654 (+2 ms)Flushing d0766cad7d1740597098828001c23b4a/B: closing flushed file at 1733897722654Flushing d0766cad7d1740597098828001c23b4a/C: creating writer at 1733897723072 (+418 ms)Flushing d0766cad7d1740597098828001c23b4a/C: appending metadata at 1733897723075 (+3 ms)Flushing d0766cad7d1740597098828001c23b4a/C: closing flushed file at 1733897723075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56a0fa41: reopening flushed file at 1733897723485 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bceb323: reopening flushed file at 1733897723490 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@589b8d31: reopening flushed file at 1733897723495 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d0766cad7d1740597098828001c23b4a in 1675ms, sequenceid=435, compaction requested=true at 1733897723500 (+5 ms)Writing region close event to WAL at 1733897723534 (+34 ms)Running coprocessor post-close hooks at 1733897723537 (+3 ms)Closed at 1733897723537 2024-12-11T06:15:23,539 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=d0766cad7d1740597098828001c23b4a, regionState=CLOSED 2024-12-11T06:15:23,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:23,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-11T06:15:23,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure d0766cad7d1740597098828001c23b4a, server=4518703b2c1d,35425,1733897658824 in 1.8680 sec 2024-12-11T06:15:23,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-12-11T06:15:23,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0766cad7d1740597098828001c23b4a, UNASSIGN in 1.8740 sec 2024-12-11T06:15:23,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-11T06:15:23,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-12-11T06:15:23,547 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897723547"}]},"ts":"1733897723547"} 2024-12-11T06:15:23,549 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:15:23,549 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:15:23,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8860 sec 2024-12-11T06:15:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T06:15:23,818 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:23,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:15:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T06:15:23,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,823 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,825 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits] 2024-12-11T06:15:23,829 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04e270e468b545aca9521eef6a07b5c1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/04e270e468b545aca9521eef6a07b5c1 2024-12-11T06:15:23,829 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/72dd01f9de18496b95ad71c8b4869eae to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/72dd01f9de18496b95ad71c8b4869eae 2024-12-11T06:15:23,829 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/6e9e5edfe9ab47898a952e32919035e6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/6e9e5edfe9ab47898a952e32919035e6 2024-12-11T06:15:23,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7a8bea4286d74f518a085a61b900bf3a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/A/7a8bea4286d74f518a085a61b900bf3a 2024-12-11T06:15:23,833 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/3e97fb44f4224613bbd6b2a15debd00f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/3e97fb44f4224613bbd6b2a15debd00f 2024-12-11T06:15:23,833 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/52538936900a4cd6971f98ae109ceaba to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/52538936900a4cd6971f98ae109ceaba 2024-12-11T06:15:23,834 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d041e69251194710bfd5a45984659b99 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d041e69251194710bfd5a45984659b99 2024-12-11T06:15:23,834 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d113ebe8197d4825bd427dd23e9abfdc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/B/d113ebe8197d4825bd427dd23e9abfdc 2024-12-11T06:15:23,837 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/47bab07613364769b9ce9ffd21f88b1e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/47bab07613364769b9ce9ffd21f88b1e 2024-12-11T06:15:23,838 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1b3afd0608a46bca461403441f7b6f5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/c1b3afd0608a46bca461403441f7b6f5 2024-12-11T06:15:23,838 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/bdfcb0f733fe411cb409f19585a0796c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/bdfcb0f733fe411cb409f19585a0796c 2024-12-11T06:15:23,838 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/a0f4e2496be24ed0899b8bc90f76acf5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/C/a0f4e2496be24ed0899b8bc90f76acf5 2024-12-11T06:15:23,841 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits/438.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a/recovered.edits/438.seqid 2024-12-11T06:15:23,842 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,842 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:15:23,842 DEBUG [PEWorker-2 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:15:23,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115bcf230ae6d543a0a64dc11e7338a6e5_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115bcf230ae6d543a0a64dc11e7338a6e5_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110b8364cf31b34bca9ad8f5ed239d88b8_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110b8364cf31b34bca9ad8f5ed239d88b8_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123279b8ed6954c55b5b282a95599724f_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123279b8ed6954c55b5b282a95599724f_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121160abddca794c4bd992df5543543ee694_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121160abddca794c4bd992df5543543ee694_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211490b9d9ca0c84f2fbf2335163d67e918_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211490b9d9ca0c84f2fbf2335163d67e918_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211642455076ef64d1c9de65a0321d1fbd7_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211642455076ef64d1c9de65a0321d1fbd7_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,852 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211789e93db56f747889eac3e499f2a8f44_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211789e93db56f747889eac3e499f2a8f44_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117ecc5cc10892497fbc1013450375800e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117ecc5cc10892497fbc1013450375800e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,853 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412118a75e0261c4f4467b7de0b80460e62be_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412118a75e0261c4f4467b7de0b80460e62be_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,854 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ad9f875f9806409087ccce8c991bda57_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ad9f875f9806409087ccce8c991bda57_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,854 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121193bf9e0211c549488dee22d9191bd68e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121193bf9e0211c549488dee22d9191bd68e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,854 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c7c9bddae4cf461593f849e1b106a05d_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c7c9bddae4cf461593f849e1b106a05d_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,854 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121191add167004045a4b71374f56fe5f284_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121191add167004045a4b71374f56fe5f284_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,854 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b582b97313a640e6b628ccac0ebe5191_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b582b97313a640e6b628ccac0ebe5191_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,855 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211924d362a0ac0448d9dbb72623229ae3e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211924d362a0ac0448d9dbb72623229ae3e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,856 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0e3727dc99545b2af1a9f05aa9c8bfd_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0e3727dc99545b2af1a9f05aa9c8bfd_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,856 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d6326844f4ac4cc396a21afee364c769_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d6326844f4ac4cc396a21afee364c769_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,857 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e34591bd3e85413a9a66a6bf58c3b77e_d0766cad7d1740597098828001c23b4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e34591bd3e85413a9a66a6bf58c3b77e_d0766cad7d1740597098828001c23b4a 2024-12-11T06:15:23,857 DEBUG [PEWorker-2 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:15:23,860 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,863 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:15:23,866 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:15:23,867 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,867 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:15:23,867 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897723867"}]},"ts":"9223372036854775807"} 2024-12-11T06:15:23,869 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:15:23,869 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d0766cad7d1740597098828001c23b4a, NAME => 'TestAcidGuarantees,,1733897697035.d0766cad7d1740597098828001c23b4a.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:15:23,869 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:15:23,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897723869"}]},"ts":"9223372036854775807"} 2024-12-11T06:15:23,871 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:15:23,871 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-12-11T06:15:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T06:15:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:15:23,929 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:23,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=257 (was 253) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-474809009_22 at /127.0.0.1:37568 [Waiting for operation #199] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1096262115_22 at /127.0.0.1:50778 [Waiting for operation #139] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1096262115_22 at /127.0.0.1:50764 [Waiting for operation #152] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-474809009_22 at /127.0.0.1:50712 [Waiting for operation #177] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=792 (was 762) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=277 (was 228) - AvailableMemoryMB LEAK? - 2024-12-11T06:15:23,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=257, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=792, ProcessCount=11, AvailableMemoryMB=277 2024-12-11T06:15:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:15:23,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:15:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:23,952 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:15:23,952 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:23,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-12-11T06:15:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T06:15:23,953 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:15:23,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742041_1217 (size=1061) 2024-12-11T06:15:23,975 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:15:23,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742042_1218 (size=53) 2024-12-11T06:15:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T06:15:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T06:15:24,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:15:24,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 682402d56c3b62fdcea61b4be01f333c, disabling compactions & flushes 2024-12-11T06:15:24,380 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. after waiting 0 ms 2024-12-11T06:15:24,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 682402d56c3b62fdcea61b4be01f333c: Waiting for close lock at 1733897724380Disabling compacts and flushes for region at 1733897724380Disabling writes for close at 1733897724381 (+1 ms)Writing region close event to WAL at 1733897724381Closed at 1733897724381 2024-12-11T06:15:24,382 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:15:24,382 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897724382"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897724382"}]},"ts":"1733897724382"} 2024-12-11T06:15:24,383 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:15:24,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:15:24,384 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897724384"}]},"ts":"1733897724384"} 2024-12-11T06:15:24,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:15:24,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, ASSIGN}] 2024-12-11T06:15:24,387 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, ASSIGN 2024-12-11T06:15:24,387 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:15:24,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=682402d56c3b62fdcea61b4be01f333c, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:24,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, ASSIGN because future has completed 2024-12-11T06:15:24,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T06:15:24,695 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,695 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:15:24,695 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,695 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:15:24,695 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,695 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,696 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,697 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:24,698 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 682402d56c3b62fdcea61b4be01f333c columnFamilyName A 2024-12-11T06:15:24,698 DEBUG [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:24,698 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(327): Store=682402d56c3b62fdcea61b4be01f333c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:24,698 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,699 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:24,699 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 682402d56c3b62fdcea61b4be01f333c columnFamilyName B 2024-12-11T06:15:24,699 DEBUG [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:24,700 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(327): Store=682402d56c3b62fdcea61b4be01f333c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:24,700 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,701 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:24,701 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 682402d56c3b62fdcea61b4be01f333c columnFamilyName C 2024-12-11T06:15:24,701 DEBUG [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:24,701 INFO [StoreOpener-682402d56c3b62fdcea61b4be01f333c-1 {}] regionserver.HStore(327): Store=682402d56c3b62fdcea61b4be01f333c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:24,702 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,702 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,702 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,703 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,703 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,703 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,704 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:15:24,705 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,707 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:15:24,707 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 682402d56c3b62fdcea61b4be01f333c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68717652, jitterRate=0.023972809314727783}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:15:24,707 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:24,708 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 682402d56c3b62fdcea61b4be01f333c: Running coprocessor pre-open hook at 1733897724695Writing region info on filesystem at 1733897724695Initializing all the Stores at 1733897724696 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897724696Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897724696Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897724696Cleaning up temporary data from old regions at 1733897724703 (+7 ms)Running coprocessor post-open hooks at 1733897724707 (+4 ms)Region opened successfully at 1733897724708 (+1 ms) 2024-12-11T06:15:24,709 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., pid=55, masterSystemTime=1733897724691 2024-12-11T06:15:24,710 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,711 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:24,711 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=682402d56c3b62fdcea61b4be01f333c, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:24,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:24,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-11T06:15:24,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 in 173 msec 2024-12-11T06:15:24,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-11T06:15:24,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, ASSIGN in 329 msec 2024-12-11T06:15:24,717 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:15:24,718 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897724718"}]},"ts":"1733897724718"} 2024-12-11T06:15:24,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:15:24,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:15:24,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 771 msec 2024-12-11T06:15:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-11T06:15:25,089 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:25,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a08a8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5831e994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76df51a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,094 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,096 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:15:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:15:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d5533da 2024-12-11T06:15:25,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:15:25,097 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:15:25,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbbb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,099 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc2fc66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ec7582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341051a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcf19f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,104 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fae8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309aa0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,105 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4833f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178a437a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6261277b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574c9c66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,111 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b11db4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b50bdc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,113 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758a8783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,115 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46568a5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,117 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616e70e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72273722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a680e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,122 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4078b8c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776317e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292fd0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5630406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db8b3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710fcbd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@116caf44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,139 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169f4509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d06c8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:25,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:25,141 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3492aedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:25,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c398318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:25,144 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,144 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,144 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,145 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,145 DEBUG [Thread-1049 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,145 DEBUG [Thread-1036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,146 DEBUG [Thread-1049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,152 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:25,155 DEBUG [Thread-1032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,156 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-11T06:15:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,157 DEBUG [Thread-1034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,159 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:25,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:15:25,165 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:25,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:25,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897785202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897785206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897785207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897785207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897785209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/6e8dcac835694b26a555142b9d3c2d28 is 50, key is test_row_0/A:col10/1733897725173/Put/seqid=0 2024-12-11T06:15:25,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742043_1219 (size=12001) 2024-12-11T06:15:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:25,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T06:15:25,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897785319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897785319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897785319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897785319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897785319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:25,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T06:15:25,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:25,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897785529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897785529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897785530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897785530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897785530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T06:15:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/6e8dcac835694b26a555142b9d3c2d28 2024-12-11T06:15:25,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/8f22dc3eb6c74086a6337fb78285664b is 50, key is test_row_0/B:col10/1733897725173/Put/seqid=0 2024-12-11T06:15:25,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742044_1220 (size=12001) 2024-12-11T06:15:25,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/8f22dc3eb6c74086a6337fb78285664b 2024-12-11T06:15:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:25,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T06:15:25,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:25,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:25,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b34878a623534a34ac4c6336478ed896 is 50, key is test_row_0/C:col10/1733897725173/Put/seqid=0 2024-12-11T06:15:25,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742045_1221 (size=12001) 2024-12-11T06:15:25,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b34878a623534a34ac4c6336478ed896 2024-12-11T06:15:25,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897785839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897785839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897785839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897785839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897785843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:25,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/6e8dcac835694b26a555142b9d3c2d28 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28 2024-12-11T06:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:25,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:25,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28, entries=150, sequenceid=13, filesize=11.7 K 2024-12-11T06:15:25,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/8f22dc3eb6c74086a6337fb78285664b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b 2024-12-11T06:15:25,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b, entries=150, sequenceid=13, filesize=11.7 K 2024-12-11T06:15:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b34878a623534a34ac4c6336478ed896 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896 2024-12-11T06:15:25,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896, entries=150, sequenceid=13, filesize=11.7 K 2024-12-11T06:15:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 682402d56c3b62fdcea61b4be01f333c in 737ms, sequenceid=13, compaction requested=false 2024-12-11T06:15:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:25,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T06:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:25,949 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:15:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84254f7ab54c4ed49cdfc761d9a2f579 is 50, key is test_row_0/A:col10/1733897725197/Put/seqid=0 2024-12-11T06:15:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742046_1222 (size=12001) 2024-12-11T06:15:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:26,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897786375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897786376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897786376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897786378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897786388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,404 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84254f7ab54c4ed49cdfc761d9a2f579 2024-12-11T06:15:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/535dfe2f8c5b494f8ab540f256507739 is 50, key is test_row_0/B:col10/1733897725197/Put/seqid=0 2024-12-11T06:15:26,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742047_1223 (size=12001) 2024-12-11T06:15:26,454 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/535dfe2f8c5b494f8ab540f256507739 2024-12-11T06:15:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f56ab391826f49ed8dee7229d46b2a7e is 50, key is test_row_0/C:col10/1733897725197/Put/seqid=0 2024-12-11T06:15:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897786489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897786489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897786489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742048_1224 (size=12001) 2024-12-11T06:15:26,509 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f56ab391826f49ed8dee7229d46b2a7e 2024-12-11T06:15:26,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84254f7ab54c4ed49cdfc761d9a2f579 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579 2024-12-11T06:15:26,533 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579, entries=150, sequenceid=33, filesize=11.7 K 2024-12-11T06:15:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/535dfe2f8c5b494f8ab540f256507739 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739 2024-12-11T06:15:26,543 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739, entries=150, sequenceid=33, filesize=11.7 K 2024-12-11T06:15:26,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f56ab391826f49ed8dee7229d46b2a7e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e 2024-12-11T06:15:26,557 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e, entries=150, sequenceid=33, filesize=11.7 K 2024-12-11T06:15:26,558 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 682402d56c3b62fdcea61b4be01f333c in 609ms, sequenceid=33, compaction requested=false 2024-12-11T06:15:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:26,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-11T06:15:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-11T06:15:26,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-11T06:15:26,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-12-11T06:15:26,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4120 sec 2024-12-11T06:15:26,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:15:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:26,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c35ad807132549cbac938e6dee0a4757 is 50, key is test_row_0/A:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:26,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742049_1225 (size=14341) 2024-12-11T06:15:26,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c35ad807132549cbac938e6dee0a4757 2024-12-11T06:15:26,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897786735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897786739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897786745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/6a8f1d1bd0774f47ad835e5ae3602bf4 is 50, key is test_row_0/B:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:26,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742050_1226 (size=12001) 2024-12-11T06:15:26,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897786848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897786848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897786858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897787059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897787061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897787068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/6a8f1d1bd0774f47ad835e5ae3602bf4 2024-12-11T06:15:27,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3d3f0fbbd5f64699ab6131d68e3db11c is 50, key is test_row_0/C:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742051_1227 (size=12001) 2024-12-11T06:15:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T06:15:27,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:27,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-11T06:15:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:27,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:27,325 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:27,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897787369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897787380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897787381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897787389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,393 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 682402d56c3b62fdcea61b4be01f333c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897787400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,403 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 682402d56c3b62fdcea61b4be01f333c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:27,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-11T06:15:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-11T06:15:27,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:27,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3d3f0fbbd5f64699ab6131d68e3db11c 2024-12-11T06:15:27,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c35ad807132549cbac938e6dee0a4757 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757 2024-12-11T06:15:27,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757, entries=200, sequenceid=50, filesize=14.0 K 2024-12-11T06:15:27,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/6a8f1d1bd0774f47ad835e5ae3602bf4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4 2024-12-11T06:15:27,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4, entries=150, sequenceid=50, filesize=11.7 K 2024-12-11T06:15:27,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3d3f0fbbd5f64699ab6131d68e3db11c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c 2024-12-11T06:15:27,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-11T06:15:27,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 682402d56c3b62fdcea61b4be01f333c in 994ms, sequenceid=50, compaction requested=true 2024-12-11T06:15:27,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:27,698 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:27,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:27,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:27,698 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:27,702 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:27,702 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:27,702 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:27,702 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:27,702 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,702 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,703 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=35.2 K 2024-12-11T06:15:27,703 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=37.4 K 2024-12-11T06:15:27,703 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f22dc3eb6c74086a6337fb78285664b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733897725171 2024-12-11T06:15:27,703 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e8dcac835694b26a555142b9d3c2d28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733897725171 2024-12-11T06:15:27,703 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 535dfe2f8c5b494f8ab540f256507739, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733897725197 2024-12-11T06:15:27,703 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84254f7ab54c4ed49cdfc761d9a2f579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733897725197 2024-12-11T06:15:27,704 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8f1d1bd0774f47ad835e5ae3602bf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:27,704 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35ad807132549cbac938e6dee0a4757, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:27,714 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:27,715 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/b0c404f0a7f74e45a5081e2c004bfea4 is 50, key is test_row_0/B:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:27,728 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:27,729 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/ce7b873f5d3847f69e2a48e91cdb8855 is 50, key is test_row_0/A:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:27,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742052_1228 (size=12104) 2024-12-11T06:15:27,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742053_1229 (size=12104) 2024-12-11T06:15:27,786 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/ce7b873f5d3847f69e2a48e91cdb8855 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/ce7b873f5d3847f69e2a48e91cdb8855 2024-12-11T06:15:27,793 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into ce7b873f5d3847f69e2a48e91cdb8855(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:27,793 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:27,793 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897727698; duration=0sec 2024-12-11T06:15:27,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-11T06:15:27,793 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:27,793 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,793 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:27,793 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:27,795 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:27,795 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:27,795 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:27,795 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=35.2 K 2024-12-11T06:15:27,796 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34878a623534a34ac4c6336478ed896, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733897725171 2024-12-11T06:15:27,796 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56ab391826f49ed8dee7229d46b2a7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733897725197 2024-12-11T06:15:27,797 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d3f0fbbd5f64699ab6131d68e3db11c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b279704365204713b9d602415ff7de53 is 50, key is test_row_0/A:col10/1733897726736/Put/seqid=0 2024-12-11T06:15:27,809 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:27,810 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/1fde8462142d49c4a669587bbe59161b is 50, key is test_row_0/C:col10/1733897726702/Put/seqid=0 2024-12-11T06:15:27,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742054_1230 (size=12001) 2024-12-11T06:15:27,835 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b279704365204713b9d602415ff7de53 2024-12-11T06:15:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/42f5c904a0a64392afc96d66b07ebf9d is 50, key is test_row_0/B:col10/1733897726736/Put/seqid=0 2024-12-11T06:15:27,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742055_1231 (size=12104) 2024-12-11T06:15:27,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742056_1232 (size=12001) 2024-12-11T06:15:27,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897787919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897787924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897787925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897788029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897788039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897788039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,188 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/b0c404f0a7f74e45a5081e2c004bfea4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/b0c404f0a7f74e45a5081e2c004bfea4 2024-12-11T06:15:28,204 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into b0c404f0a7f74e45a5081e2c004bfea4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:28,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:28,204 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897727698; duration=0sec 2024-12-11T06:15:28,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:28,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:28,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897788241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897788252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897788252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,267 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/1fde8462142d49c4a669587bbe59161b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/1fde8462142d49c4a669587bbe59161b 2024-12-11T06:15:28,276 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 1fde8462142d49c4a669587bbe59161b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:28,276 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:28,276 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897727699; duration=0sec 2024-12-11T06:15:28,276 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:28,276 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:28,281 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/42f5c904a0a64392afc96d66b07ebf9d 2024-12-11T06:15:28,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/64ae19d0d3e840709ac43fe40d70cd0f is 50, key is test_row_0/C:col10/1733897726736/Put/seqid=0 2024-12-11T06:15:28,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742057_1233 (size=12001) 2024-12-11T06:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897788558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897788569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897788569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:28,695 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/64ae19d0d3e840709ac43fe40d70cd0f 2024-12-11T06:15:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b279704365204713b9d602415ff7de53 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53 2024-12-11T06:15:28,704 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53, entries=150, sequenceid=66, filesize=11.7 K 2024-12-11T06:15:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/42f5c904a0a64392afc96d66b07ebf9d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d 2024-12-11T06:15:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d, entries=150, sequenceid=66, filesize=11.7 K 2024-12-11T06:15:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/64ae19d0d3e840709ac43fe40d70cd0f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f 2024-12-11T06:15:28,715 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f, entries=150, sequenceid=66, filesize=11.7 K 2024-12-11T06:15:28,716 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 682402d56c3b62fdcea61b4be01f333c in 923ms, sequenceid=66, compaction requested=false 2024-12-11T06:15:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-11T06:15:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-11T06:15:28,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-11T06:15:28,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-12-11T06:15:28,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.3970 sec 2024-12-11T06:15:29,058 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:29,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7aa812dfeed44a96be6b40fd001a2448 is 50, key is test_row_0/A:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:29,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742058_1234 (size=12001) 2024-12-11T06:15:29,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7aa812dfeed44a96be6b40fd001a2448 2024-12-11T06:15:29,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/957d29a66ec34baabcbcd665a7ebf0d7 is 50, key is test_row_0/B:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897789094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742059_1235 (size=12001) 2024-12-11T06:15:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897789095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897789096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897789209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897789209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897789209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897789408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,411 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 682402d56c3b62fdcea61b4be01f333c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897789418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897789419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897789419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,421 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 682402d56c3b62fdcea61b4be01f333c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897789419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T06:15:29,479 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:29,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-11T06:15:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-11T06:15:29,480 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:29,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:29,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:29,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/957d29a66ec34baabcbcd665a7ebf0d7 2024-12-11T06:15:29,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/09774403dfa14645bd29eb97ad8b4f8d is 50, key is test_row_0/C:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:29,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742060_1236 (size=12001) 2024-12-11T06:15:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-11T06:15:29,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-11T06:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897789728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897789728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897789728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:29,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-11T06:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-11T06:15:29,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/09774403dfa14645bd29eb97ad8b4f8d 2024-12-11T06:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7aa812dfeed44a96be6b40fd001a2448 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448 2024-12-11T06:15:29,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448, entries=150, sequenceid=86, filesize=11.7 K 2024-12-11T06:15:29,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/957d29a66ec34baabcbcd665a7ebf0d7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7 2024-12-11T06:15:29,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7, entries=150, sequenceid=86, filesize=11.7 K 2024-12-11T06:15:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/09774403dfa14645bd29eb97ad8b4f8d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d 2024-12-11T06:15:29,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d, entries=150, sequenceid=86, filesize=11.7 K 2024-12-11T06:15:29,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 682402d56c3b62fdcea61b4be01f333c in 865ms, sequenceid=86, compaction requested=true 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:29,935 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:29,935 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:29,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:29,936 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:29,936 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:29,936 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:29,936 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:29,936 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,936 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,936 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/ce7b873f5d3847f69e2a48e91cdb8855, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=35.3 K 2024-12-11T06:15:29,936 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/b0c404f0a7f74e45a5081e2c004bfea4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=35.3 K 2024-12-11T06:15:29,937 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c404f0a7f74e45a5081e2c004bfea4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:29,937 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7b873f5d3847f69e2a48e91cdb8855, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:29,937 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f5c904a0a64392afc96d66b07ebf9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733897726729 2024-12-11T06:15:29,937 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b279704365204713b9d602415ff7de53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733897726729 2024-12-11T06:15:29,937 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa812dfeed44a96be6b40fd001a2448, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:29,938 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 957d29a66ec34baabcbcd665a7ebf0d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:29,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:29,939 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0ec078b5d431490a850d7ed43018c064 is 50, key is test_row_0/A:col10/1733897729094/Put/seqid=0 2024-12-11T06:15:29,946 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:29,946 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:29,947 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/6e38e7e116954a5a8f868a7f46b24ef5 is 50, key is test_row_0/A:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:29,947 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2ba98f02ae34429dbecbd2465499ff63 is 50, key is test_row_0/B:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:29,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742061_1237 (size=9657) 2024-12-11T06:15:29,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742062_1238 (size=12207) 2024-12-11T06:15:29,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742063_1239 (size=12207) 2024-12-11T06:15:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-11T06:15:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:30,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897790277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897790277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897790279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,359 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0ec078b5d431490a850d7ed43018c064 2024-12-11T06:15:30,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/bc7f5c9457724213a6d6bd3abac045b0 is 50, key is test_row_0/B:col10/1733897729094/Put/seqid=0 2024-12-11T06:15:30,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742064_1240 (size=9657) 2024-12-11T06:15:30,375 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/bc7f5c9457724213a6d6bd3abac045b0 2024-12-11T06:15:30,379 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/6e38e7e116954a5a8f868a7f46b24ef5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e38e7e116954a5a8f868a7f46b24ef5 2024-12-11T06:15:30,391 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2ba98f02ae34429dbecbd2465499ff63 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2ba98f02ae34429dbecbd2465499ff63 2024-12-11T06:15:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897790388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,393 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into 6e38e7e116954a5a8f868a7f46b24ef5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,393 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,393 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897729935; duration=0sec 2024-12-11T06:15:30,393 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:30,393 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:30,393 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:30,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897790388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897790389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,395 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:30,395 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:30,395 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,396 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/1fde8462142d49c4a669587bbe59161b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=35.3 K 2024-12-11T06:15:30,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d6b8e6a43a674aeba3c3d0beaf1c616e is 50, key is test_row_0/C:col10/1733897729094/Put/seqid=0 2024-12-11T06:15:30,396 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fde8462142d49c4a669587bbe59161b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733897726371 2024-12-11T06:15:30,397 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ae19d0d3e840709ac43fe40d70cd0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733897726729 2024-12-11T06:15:30,400 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 2ba98f02ae34429dbecbd2465499ff63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:30,400 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:30,400 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897729935; duration=0sec 2024-12-11T06:15:30,400 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:30,400 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:30,401 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09774403dfa14645bd29eb97ad8b4f8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:30,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742065_1241 (size=9657) 2024-12-11T06:15:30,410 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d6b8e6a43a674aeba3c3d0beaf1c616e 2024-12-11T06:15:30,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0ec078b5d431490a850d7ed43018c064 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064 2024-12-11T06:15:30,417 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:30,417 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/affa960e214740dbb6784512c40c5be9 is 50, key is test_row_0/C:col10/1733897727920/Put/seqid=0 2024-12-11T06:15:30,433 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064, entries=100, sequenceid=100, filesize=9.4 K 2024-12-11T06:15:30,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/bc7f5c9457724213a6d6bd3abac045b0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0 2024-12-11T06:15:30,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742066_1242 (size=12207) 2024-12-11T06:15:30,441 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0, entries=100, sequenceid=100, filesize=9.4 K 2024-12-11T06:15:30,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d6b8e6a43a674aeba3c3d0beaf1c616e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e 2024-12-11T06:15:30,449 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/affa960e214740dbb6784512c40c5be9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/affa960e214740dbb6784512c40c5be9 2024-12-11T06:15:30,449 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e, entries=100, sequenceid=100, filesize=9.4 K 2024-12-11T06:15:30,451 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 682402d56c3b62fdcea61b4be01f333c in 512ms, sequenceid=100, compaction requested=false 2024-12-11T06:15:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-11T06:15:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-11T06:15:30,456 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into affa960e214740dbb6784512c40c5be9(size=11.9 K), total size for store is 21.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:30,456 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:30,456 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897729935; duration=0sec 2024-12-11T06:15:30,456 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:30,457 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:30,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-11T06:15:30,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 975 msec 2024-12-11T06:15:30,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 982 msec 2024-12-11T06:15:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:30,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:30,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897730274, max timestamp=1733897730600Num uniques 859; 2024-12-11T06:15:30,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897730274, max timestamp=1733897730600Num uniques 859; 2024-12-11T06:15:30,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:30,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:30,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:30,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/e3afb47d7bf4492d89763281768fb345 is 50, key is test_row_0/A:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-11T06:15:30,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:30,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-11T06:15:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742067_1243 (size=16681) 2024-12-11T06:15:30,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:30,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:30,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:30,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897790640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897790643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897790644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:30,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897790749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897790749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897790758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:30,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:30,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897790958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897790959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:30,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897790969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/e3afb47d7bf4492d89763281768fb345 2024-12-11T06:15:31,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/ac48f8b3c3834ca3823177f9f383becf is 50, key is test_row_0/B:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:31,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742068_1244 (size=12001) 2024-12-11T06:15:31,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/ac48f8b3c3834ca3823177f9f383becf 2024-12-11T06:15:31,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9cc750aeb27d44eb851d93f5e1d33a2f is 50, key is test_row_0/C:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:31,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742069_1245 (size=12001) 2024-12-11T06:15:31,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:31,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897791268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897791268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897791278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:31,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9cc750aeb27d44eb851d93f5e1d33a2f 2024-12-11T06:15:31,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/e3afb47d7bf4492d89763281768fb345 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345 2024-12-11T06:15:31,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345, entries=250, sequenceid=125, filesize=16.3 K 2024-12-11T06:15:31,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/ac48f8b3c3834ca3823177f9f383becf as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf 2024-12-11T06:15:31,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf, entries=150, sequenceid=125, filesize=11.7 K 2024-12-11T06:15:31,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9cc750aeb27d44eb851d93f5e1d33a2f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f 2024-12-11T06:15:31,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f, entries=150, sequenceid=125, filesize=11.7 K 2024-12-11T06:15:31,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 682402d56c3b62fdcea61b4be01f333c in 875ms, sequenceid=125, compaction requested=true 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:31,478 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:31,478 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:31,479 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:31,479 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:31,479 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,479 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e38e7e116954a5a8f868a7f46b24ef5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=37.6 K 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e38e7e116954a5a8f868a7f46b24ef5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:31,480 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,480 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2ba98f02ae34429dbecbd2465499ff63, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=33.1 K 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec078b5d431490a850d7ed43018c064, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733897729093 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba98f02ae34429dbecbd2465499ff63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3afb47d7bf4492d89763281768fb345, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:31,480 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7f5c9457724213a6d6bd3abac045b0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733897729093 2024-12-11T06:15:31,481 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ac48f8b3c3834ca3823177f9f383becf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:31,487 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:31,487 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:31,487 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b3f91397ba3943b087913bdc1f419544 is 50, key is test_row_0/A:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:31,487 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/09112a70726647a99e7b317360d593cb is 50, key is test_row_0/B:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:31,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742070_1246 (size=12309) 2024-12-11T06:15:31,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742071_1247 (size=12309) 2024-12-11T06:15:31,498 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/09112a70726647a99e7b317360d593cb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/09112a70726647a99e7b317360d593cb 2024-12-11T06:15:31,503 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 09112a70726647a99e7b317360d593cb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:31,503 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:31,503 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897731478; duration=0sec 2024-12-11T06:15:31,503 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:31,503 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:31,503 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:31,504 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:31,504 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:31,504 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,504 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/affa960e214740dbb6784512c40c5be9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=33.1 K 2024-12-11T06:15:31,504 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting affa960e214740dbb6784512c40c5be9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733897727920 2024-12-11T06:15:31,505 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b8e6a43a674aeba3c3d0beaf1c616e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733897729093 2024-12-11T06:15:31,505 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc750aeb27d44eb851d93f5e1d33a2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:31,512 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:31,513 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b739b104c85d42f6b2819156efb8a888 is 50, key is test_row_0/C:col10/1733897730600/Put/seqid=0 2024-12-11T06:15:31,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742072_1248 (size=12309) 2024-12-11T06:15:31,528 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b739b104c85d42f6b2819156efb8a888 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b739b104c85d42f6b2819156efb8a888 2024-12-11T06:15:31,532 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into b739b104c85d42f6b2819156efb8a888(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:31,533 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:31,533 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897731478; duration=0sec 2024-12-11T06:15:31,533 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:31,533 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:31,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-11T06:15:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:31,570 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b24179c634d546b09ba3b029e59125ee is 50, key is test_row_0/A:col10/1733897730643/Put/seqid=0 2024-12-11T06:15:31,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742073_1249 (size=12151) 2024-12-11T06:15:31,582 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b24179c634d546b09ba3b029e59125ee 2024-12-11T06:15:31,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fcb8f60e032b4ae09902dea782b5ac09 is 50, key is test_row_0/B:col10/1733897730643/Put/seqid=0 2024-12-11T06:15:31,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742074_1250 (size=12151) 2024-12-11T06:15:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:31,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897791792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897791796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897791796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897791909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897791909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,915 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b3f91397ba3943b087913bdc1f419544 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b3f91397ba3943b087913bdc1f419544 2024-12-11T06:15:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897791909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:31,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:31,922 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into b3f91397ba3943b087913bdc1f419544(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:31,922 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:31,922 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897731478; duration=0sec 2024-12-11T06:15:31,922 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:31,922 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:31,996 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fcb8f60e032b4ae09902dea782b5ac09 2024-12-11T06:15:32,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9d4108006637465f92606a48c90f9eb2 is 50, key is test_row_0/C:col10/1733897730643/Put/seqid=0 2024-12-11T06:15:32,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742075_1251 (size=12151) 2024-12-11T06:15:32,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897792118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897792118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897792129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,421 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9d4108006637465f92606a48c90f9eb2 2024-12-11T06:15:32,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897792429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897792429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b24179c634d546b09ba3b029e59125ee as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee 2024-12-11T06:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,436 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee, entries=150, sequenceid=147, filesize=11.9 K 2024-12-11T06:15:32,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fcb8f60e032b4ae09902dea782b5ac09 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09 2024-12-11T06:15:32,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897792438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,441 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09, entries=150, sequenceid=147, filesize=11.9 K 2024-12-11T06:15:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/9d4108006637465f92606a48c90f9eb2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2 2024-12-11T06:15:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2, entries=150, sequenceid=147, filesize=11.9 K 2024-12-11T06:15:32,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 682402d56c3b62fdcea61b4be01f333c in 884ms, sequenceid=147, compaction requested=false 2024-12-11T06:15:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-11T06:15:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-11T06:15:32,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-11T06:15:32,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-12-11T06:15:32,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8390 sec 2024-12-11T06:15:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T06:15:32,769 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:32,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-11T06:15:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-11T06:15:32,772 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:32,773 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:32,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-11T06:15:32,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-11T06:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:32,929 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:32,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/07e23f936f67473595ead9bdc03d51a0 is 50, key is test_row_0/A:col10/1733897731794/Put/seqid=0 2024-12-11T06:15:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:32,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:32,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742076_1252 (size=12151) 2024-12-11T06:15:32,978 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/07e23f936f67473595ead9bdc03d51a0 2024-12-11T06:15:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2f27978e35a044b89bfc5e6e81b0f77e is 50, key is test_row_0/B:col10/1733897731794/Put/seqid=0 2024-12-11T06:15:32,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897792992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897792992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:32,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897792995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742077_1253 (size=12151) 2024-12-11T06:15:33,006 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2f27978e35a044b89bfc5e6e81b0f77e 2024-12-11T06:15:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/27f76ee04597464d9c700ea3f02af394 is 50, key is test_row_0/C:col10/1733897731794/Put/seqid=0 2024-12-11T06:15:33,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742078_1254 (size=12151) 2024-12-11T06:15:33,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/27f76ee04597464d9c700ea3f02af394 2024-12-11T06:15:33,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/07e23f936f67473595ead9bdc03d51a0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0 2024-12-11T06:15:33,067 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0, entries=150, sequenceid=160, filesize=11.9 K 2024-12-11T06:15:33,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2f27978e35a044b89bfc5e6e81b0f77e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e 2024-12-11T06:15:33,076 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e, entries=150, sequenceid=160, filesize=11.9 K 2024-12-11T06:15:33,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/27f76ee04597464d9c700ea3f02af394 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394 2024-12-11T06:15:33,083 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394, entries=150, sequenceid=160, filesize=11.9 K 2024-12-11T06:15:33,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 682402d56c3b62fdcea61b4be01f333c in 155ms, sequenceid=160, compaction requested=true 2024-12-11T06:15:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-11T06:15:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-11T06:15:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-11T06:15:33,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-11T06:15:33,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 313 msec 2024-12-11T06:15:33,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 321 msec 2024-12-11T06:15:33,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-11T06:15:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897732975, max timestamp=1733897733102Num uniques 859; 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897732975, max timestamp=1733897733102Num uniques 859; 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897732975, max timestamp=1733897733102Num uniques 859; 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:33,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:33,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:33,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c1314643c09642548367e686bf21b248 is 50, key is test_row_0/A:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:33,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742079_1255 (size=14541) 2024-12-11T06:15:33,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c1314643c09642548367e686bf21b248 2024-12-11T06:15:33,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/02fdde8a247342fc9eb6654c69fe1eda is 50, key is test_row_0/B:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897733156, max timestamp=1733897733174Num uniques 859; 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897733156, max timestamp=1733897733174Num uniques 859; 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897733156, max timestamp=1733897733174Num uniques 859; 2024-12-11T06:15:33,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742080_1256 (size=12151) 2024-12-11T06:15:33,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897733243, max timestamp=1733897733260Num uniques 1068; 2024-12-11T06:15:33,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897733243, max timestamp=1733897733260Num uniques 1068; 2024-12-11T06:15:33,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897733243, max timestamp=1733897733260Num uniques 1068; 2024-12-11T06:15:33,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-11T06:15:33,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-11T06:15:33,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-11T06:15:33,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897733296, max timestamp=1733897733309Num uniques 1027; 2024-12-11T06:15:33,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:33,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897733296, max timestamp=1733897733309Num uniques 1027; 2024-12-11T06:15:33,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897733296, max timestamp=1733897733309Num uniques 1027; 2024-12-11T06:15:33,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:33,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:33,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733897733345, max timestamp=1733897733369Num uniques 1036; 2024-12-11T06:15:33,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-11T06:15:33,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733897733345, max timestamp=1733897733369Num uniques 1036; 2024-12-11T06:15:33,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-11T06:15:33,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897733345, max timestamp=1733897733369Num uniques 1036; 2024-12-11T06:15:33,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-11T06:15:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-11T06:15:33,398 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:33,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-11T06:15:33,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:33,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:33,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:33,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733897733402, max timestamp=1733897733432Num uniques 1045; 2024-12-11T06:15:33,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733897733402, max timestamp=1733897733432Num uniques 1045; 2024-12-11T06:15:33,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-12-11T06:15:33,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-11T06:15:33,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733897733402, max timestamp=1733897733432Num uniques 1009; 2024-12-11T06:15:33,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-12-11T06:15:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733897733463, max timestamp=1733897733481Num uniques 1204; 2024-12-11T06:15:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-12-11T06:15:33,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897733463, max timestamp=1733897733481Num uniques 1054; 2024-12-11T06:15:33,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-11T06:15:33,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897733463, max timestamp=1733897733481Num uniques 1054; 2024-12-11T06:15:33,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-11T06:15:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897793519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733897733514, max timestamp=1733897733524Num uniques 1063; 2024-12-11T06:15:33,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733897733514, max timestamp=1733897733524Num uniques 1013; 2024-12-11T06:15:33,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-11T06:15:33,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-11T06:15:33,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897733514, max timestamp=1733897733524Num uniques 1013; 2024-12-11T06:15:33,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=6870, suffix heap size=133320,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=150 2024-12-11T06:15:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T06:15:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:33,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733897733549, max timestamp=1733897733568Num uniques 1072; 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733897733549, max timestamp=1733897733568Num uniques 1072; 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733897733549, max timestamp=1733897733568Num uniques 1009; 2024-12-11T06:15:33,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12452, suffix heap size=132904,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=272 2024-12-11T06:15:33,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/02fdde8a247342fc9eb6654c69fe1eda 2024-12-11T06:15:33,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733897733599, max timestamp=1733897733625Num uniques 1131; 2024-12-11T06:15:33,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-11T06:15:33,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/41af954225724470aa26d5d8e3eb924b is 50, key is test_row_0/C:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:33,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733897733599, max timestamp=1733897733625Num uniques 1131; 2024-12-11T06:15:33,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-11T06:15:33,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733897733599, max timestamp=1733897733625Num uniques 1131; 2024-12-11T06:15:33,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733897733645, max timestamp=1733897733667Num uniques 1090; 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733897733645, max timestamp=1733897733667Num uniques 1090; 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=340, cellSize=15570, totalHeapSize=35600, min timestamp=1733897733645, max timestamp=1733897733667Num uniques 1090; 2024-12-11T06:15:33,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=15570, suffix heap size=141328,new segment heap size=35600 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=340 2024-12-11T06:15:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742081_1257 (size=12151) 2024-12-11T06:15:33,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733897733687, max timestamp=1733897733698Num uniques 1199; 2024-12-11T06:15:33,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-11T06:15:33,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733897733687, max timestamp=1733897733698Num uniques 1199; 2024-12-11T06:15:33,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-11T06:15:33,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733897733687, max timestamp=1733897733698Num uniques 1199; 2024-12-11T06:15:33,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54914, new segment data size=11404, suffix heap size=152664,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1199, new segment cells count=249 2024-12-11T06:15:33,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T06:15:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:33,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733897733728, max timestamp=1733897733737Num uniques 1108; 2024-12-11T06:15:33,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-11T06:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733897733728, max timestamp=1733897733737Num uniques 1108; 2024-12-11T06:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-11T06:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1733897733728, max timestamp=1733897733737Num uniques 1108; 2024-12-11T06:15:33,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-12-11T06:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1733897733751, max timestamp=1733897733771Num uniques 1117; 2024-12-11T06:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=16802, suffix heap size=144136,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=367 2024-12-11T06:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1733897733751, max timestamp=1733897733771Num uniques 1117; 2024-12-11T06:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=16802, suffix heap size=144136,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=367 2024-12-11T06:15:33,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=367, cellSize=16802, totalHeapSize=38408, min timestamp=1733897733751, max timestamp=1733897733771Num uniques 1117; 2024-12-11T06:15:33,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=16802, suffix heap size=144136,new segment heap size=38408 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=367 2024-12-11T06:15:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897793802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897793804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897793806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897793808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897793810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T06:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:33,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:33,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897793918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897793919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897793919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897793919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897793919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:33,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T06:15:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:34,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/41af954225724470aa26d5d8e3eb924b 2024-12-11T06:15:34,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c1314643c09642548367e686bf21b248 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248 2024-12-11T06:15:34,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248, entries=200, sequenceid=182, filesize=14.2 K 2024-12-11T06:15:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/02fdde8a247342fc9eb6654c69fe1eda as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda 2024-12-11T06:15:34,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda, entries=150, sequenceid=182, filesize=11.9 K 2024-12-11T06:15:34,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/41af954225724470aa26d5d8e3eb924b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b 2024-12-11T06:15:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897794128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897794128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897794129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897794129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897794129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b, entries=150, sequenceid=182, filesize=11.9 K 2024-12-11T06:15:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for 682402d56c3b62fdcea61b4be01f333c in 1030ms, sequenceid=182, compaction requested=true 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:34,135 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:34,135 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:34,137 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:34,137 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:34,137 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,137 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b3f91397ba3943b087913bdc1f419544, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=50.0 K 2024-12-11T06:15:34,137 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:34,137 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:34,138 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,138 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/09112a70726647a99e7b317360d593cb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=47.6 K 2024-12-11T06:15:34,139 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 09112a70726647a99e7b317360d593cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:34,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f91397ba3943b087913bdc1f419544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:34,140 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb8f60e032b4ae09902dea782b5ac09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897730638 2024-12-11T06:15:34,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24179c634d546b09ba3b029e59125ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897730638 2024-12-11T06:15:34,140 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f27978e35a044b89bfc5e6e81b0f77e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897731790 2024-12-11T06:15:34,141 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e23f936f67473595ead9bdc03d51a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897731790 2024-12-11T06:15:34,141 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fdde8a247342fc9eb6654c69fe1eda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:34,141 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1314643c09642548367e686bf21b248, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:34,163 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:34,164 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/f9a634e32837464aa02e980889b720cf is 50, key is test_row_0/B:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:34,173 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:34,173 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/a7d305a55dc14b949f13770d647b0f4c is 50, key is test_row_0/A:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:34,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,176 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=161.02 KB heapSize=445.17 KB 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/23ce0c00f8574f0db396c0e7b1861878 is 50, key is test_row_0/A:col10/1733897733804/Put/seqid=0 2024-12-11T06:15:34,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742083_1259 (size=12595) 2024-12-11T06:15:34,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742084_1260 (size=12301) 2024-12-11T06:15:34,240 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/23ce0c00f8574f0db396c0e7b1861878 2024-12-11T06:15:34,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742082_1258 (size=12595) 2024-12-11T06:15:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4305ca7be70444e1b3a3bb1eb2c579c9 is 50, key is test_row_0/B:col10/1733897733804/Put/seqid=0 2024-12-11T06:15:34,255 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/f9a634e32837464aa02e980889b720cf as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/f9a634e32837464aa02e980889b720cf 2024-12-11T06:15:34,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742085_1261 (size=12301) 2024-12-11T06:15:34,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4305ca7be70444e1b3a3bb1eb2c579c9 2024-12-11T06:15:34,261 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into f9a634e32837464aa02e980889b720cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:34,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:34,262 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=12, startTime=1733897734135; duration=0sec 2024-12-11T06:15:34,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:34,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:34,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/ea6f76bb7e304d0d926a898fe980afce is 50, key is test_row_0/C:col10/1733897733804/Put/seqid=0 2024-12-11T06:15:34,264 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:34,264 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:34,264 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,264 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b739b104c85d42f6b2819156efb8a888, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=47.6 K 2024-12-11T06:15:34,265 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b739b104c85d42f6b2819156efb8a888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733897730274 2024-12-11T06:15:34,269 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4108006637465f92606a48c90f9eb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897730638 2024-12-11T06:15:34,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742086_1262 (size=12301) 2024-12-11T06:15:34,269 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 27f76ee04597464d9c700ea3f02af394, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733897731790 2024-12-11T06:15:34,270 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 41af954225724470aa26d5d8e3eb924b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:34,281 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:34,281 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/7cdcce4a47fa4c1e81774e87880bbd3c is 50, key is test_row_0/C:col10/1733897733103/Put/seqid=0 2024-12-11T06:15:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742087_1263 (size=12595) 2024-12-11T06:15:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897794444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897794445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897794445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897794446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897794447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:34,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897794558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897794559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897794559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897794559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897794559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,633 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/a7d305a55dc14b949f13770d647b0f4c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/a7d305a55dc14b949f13770d647b0f4c 2024-12-11T06:15:34,642 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into a7d305a55dc14b949f13770d647b0f4c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:34,642 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:34,643 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=12, startTime=1733897734135; duration=0sec 2024-12-11T06:15:34,643 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:34,643 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:34,669 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/ea6f76bb7e304d0d926a898fe980afce 2024-12-11T06:15:34,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/23ce0c00f8574f0db396c0e7b1861878 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878 2024-12-11T06:15:34,681 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878, entries=150, sequenceid=425, filesize=12.0 K 2024-12-11T06:15:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4305ca7be70444e1b3a3bb1eb2c579c9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9 2024-12-11T06:15:34,691 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9, entries=150, sequenceid=425, filesize=12.0 K 2024-12-11T06:15:34,693 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/7cdcce4a47fa4c1e81774e87880bbd3c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/7cdcce4a47fa4c1e81774e87880bbd3c 2024-12-11T06:15:34,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/ea6f76bb7e304d0d926a898fe980afce as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce 2024-12-11T06:15:34,703 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 7cdcce4a47fa4c1e81774e87880bbd3c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:34,703 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:34,703 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=12, startTime=1733897734135; duration=0sec 2024-12-11T06:15:34,704 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:34,704 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:34,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce, entries=150, sequenceid=425, filesize=12.0 K 2024-12-11T06:15:34,708 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.13 KB/455808, currentSize=33.54 KB/34350 for 682402d56c3b62fdcea61b4be01f333c in 532ms, sequenceid=425, compaction requested=false 2024-12-11T06:15:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-11T06:15:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-11T06:15:34,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-11T06:15:34,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3080 sec 2024-12-11T06:15:34,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3180 sec 2024-12-11T06:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:34,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:34,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:34,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/08211b4df9514effa32b940872cfd59d is 50, key is test_row_0/A:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897794793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897734789, max timestamp=1733897734797Num uniques 859; 2024-12-11T06:15:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897734789, max timestamp=1733897734797Num uniques 859; 2024-12-11T06:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:15:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897794795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:15:34,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897734792, max timestamp=1733897734797Num uniques 859; 2024-12-11T06:15:34,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742088_1264 (size=12301) 2024-12-11T06:15:34,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897794830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897794830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897794830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897794908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897794914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897794948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897794948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897794949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897795119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897795119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897795159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897795160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897795160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/08211b4df9514effa32b940872cfd59d 2024-12-11T06:15:35,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/57a31a35a84c404cbef5aef55b90e8b0 is 50, key is test_row_0/B:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:35,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742089_1265 (size=12301) 2024-12-11T06:15:35,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/57a31a35a84c404cbef5aef55b90e8b0 2024-12-11T06:15:35,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/2447db1c0df44c8b9d8184d7754f0d27 is 50, key is test_row_0/C:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:35,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742090_1266 (size=12301) 2024-12-11T06:15:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897795429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897795429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897795469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897795469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897795469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T06:15:35,549 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:35,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-11T06:15:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T06:15:35,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:35,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:35,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T06:15:35,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/2447db1c0df44c8b9d8184d7754f0d27 2024-12-11T06:15:35,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/08211b4df9514effa32b940872cfd59d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d 2024-12-11T06:15:35,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d, entries=150, sequenceid=439, filesize=12.0 K 2024-12-11T06:15:35,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/57a31a35a84c404cbef5aef55b90e8b0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0 2024-12-11T06:15:35,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0, entries=150, sequenceid=439, filesize=12.0 K 2024-12-11T06:15:35,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/2447db1c0df44c8b9d8184d7754f0d27 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27 2024-12-11T06:15:35,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27, entries=150, sequenceid=439, filesize=12.0 K 2024-12-11T06:15:35,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 682402d56c3b62fdcea61b4be01f333c in 918ms, sequenceid=439, compaction requested=true 2024-12-11T06:15:35,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:35,692 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:35,692 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:35,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:35,693 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:35,693 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:35,693 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:35,693 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/f9a634e32837464aa02e980889b720cf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.3 K 2024-12-11T06:15:35,695 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:35,695 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:35,695 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:35,695 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/a7d305a55dc14b949f13770d647b0f4c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.3 K 2024-12-11T06:15:35,695 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a634e32837464aa02e980889b720cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:35,695 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d305a55dc14b949f13770d647b0f4c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:35,696 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 4305ca7be70444e1b3a3bb1eb2c579c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733897733795 2024-12-11T06:15:35,696 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ce0c00f8574f0db396c0e7b1861878, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733897733795 2024-12-11T06:15:35,696 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a31a35a84c404cbef5aef55b90e8b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:35,696 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08211b4df9514effa32b940872cfd59d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:35,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T06:15:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:35,704 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:35,705 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:35,706 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/9cc6ce8481804732813ad88d1785ee58 is 50, key is test_row_0/B:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:35,708 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:35,709 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/708a7da8e6024ed399687a8a9bda56b5 is 50, key is test_row_0/A:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b4bb8aad70a0422caffd47ce0f6a715b is 50, key is test_row_0/A:col10/1733897734828/Put/seqid=0 2024-12-11T06:15:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742091_1267 (size=12847) 2024-12-11T06:15:35,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742092_1268 (size=12847) 2024-12-11T06:15:35,743 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/9cc6ce8481804732813ad88d1785ee58 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/9cc6ce8481804732813ad88d1785ee58 2024-12-11T06:15:35,746 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/708a7da8e6024ed399687a8a9bda56b5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/708a7da8e6024ed399687a8a9bda56b5 2024-12-11T06:15:35,749 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 9cc6ce8481804732813ad88d1785ee58(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:35,749 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:35,749 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897735692; duration=0sec 2024-12-11T06:15:35,749 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:35,749 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:35,749 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:35,751 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:35,751 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:35,751 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:35,751 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/7cdcce4a47fa4c1e81774e87880bbd3c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.3 K 2024-12-11T06:15:35,751 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cdcce4a47fa4c1e81774e87880bbd3c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733897732975 2024-12-11T06:15:35,752 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into 708a7da8e6024ed399687a8a9bda56b5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:35,752 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:35,752 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897735692; duration=0sec 2024-12-11T06:15:35,752 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:35,752 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:35,752 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6f76bb7e304d0d926a898fe980afce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733897733795 2024-12-11T06:15:35,752 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2447db1c0df44c8b9d8184d7754f0d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:35,758 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:35,759 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/5d3b1accbd0646c9b1ecaa186d121769 is 50, key is test_row_0/C:col10/1733897734772/Put/seqid=0 2024-12-11T06:15:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742093_1269 (size=12301) 2024-12-11T06:15:35,765 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b4bb8aad70a0422caffd47ce0f6a715b 2024-12-11T06:15:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/162d3b8899b04235a79b40b04ab8a487 is 50, key is test_row_0/B:col10/1733897734828/Put/seqid=0 2024-12-11T06:15:35,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742094_1270 (size=12847) 2024-12-11T06:15:35,799 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/5d3b1accbd0646c9b1ecaa186d121769 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/5d3b1accbd0646c9b1ecaa186d121769 2024-12-11T06:15:35,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742095_1271 (size=12301) 2024-12-11T06:15:35,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/162d3b8899b04235a79b40b04ab8a487 2024-12-11T06:15:35,804 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 5d3b1accbd0646c9b1ecaa186d121769(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:35,804 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:35,804 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897735692; duration=0sec 2024-12-11T06:15:35,804 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:35,804 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/4adba295cb4949aaab35631a24c04fcd is 50, key is test_row_0/C:col10/1733897734828/Put/seqid=0 2024-12-11T06:15:35,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742096_1272 (size=12301) 2024-12-11T06:15:35,814 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/4adba295cb4949aaab35631a24c04fcd 2024-12-11T06:15:35,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b4bb8aad70a0422caffd47ce0f6a715b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b 2024-12-11T06:15:35,831 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b, entries=150, sequenceid=475, filesize=12.0 K 2024-12-11T06:15:35,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/162d3b8899b04235a79b40b04ab8a487 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487 2024-12-11T06:15:35,842 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487, entries=150, sequenceid=475, filesize=12.0 K 2024-12-11T06:15:35,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/4adba295cb4949aaab35631a24c04fcd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd 2024-12-11T06:15:35,854 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd, entries=150, sequenceid=475, filesize=12.0 K 2024-12-11T06:15:35,855 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=0 B/0 for 682402d56c3b62fdcea61b4be01f333c in 152ms, sequenceid=475, compaction requested=false 2024-12-11T06:15:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-11T06:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-11T06:15:35,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-11T06:15:35,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 306 msec 2024-12-11T06:15:35,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 312 msec 2024-12-11T06:15:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T06:15:35,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:35,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-11T06:15:35,872 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:35,873 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:35,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:15:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:35,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:35,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:35,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4984229695fb44efa7415835b619c3bf is 50, key is test_row_0/A:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:35,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742097_1273 (size=17181) 2024-12-11T06:15:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897795981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733897735971, max timestamp=1733897735984Num uniques 859; 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733897735971, max timestamp=1733897735984Num uniques 859; 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733897735971, max timestamp=1733897735984Num uniques 859; 2024-12-11T06:15:35,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897796007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897796010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897796012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897796013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897796089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897796118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897796118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897796119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897796129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897796299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897796330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897796330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897796330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897796338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4984229695fb44efa7415835b619c3bf 2024-12-11T06:15:36,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/706c0e9a09994da083ea5d0919a9da8a is 50, key is test_row_0/B:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:36,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742098_1274 (size=12301) 2024-12-11T06:15:36,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:36,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897796618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897796639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897796648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897796648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897796648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:36,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/706c0e9a09994da083ea5d0919a9da8a 2024-12-11T06:15:36,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/eb40a5f043a84411b83aeb95836b37d4 is 50, key is test_row_0/C:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:36,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742099_1275 (size=12301) 2024-12-11T06:15:36,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:36,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:37,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:37,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:37,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897797129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897797148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:37,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897797158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897797158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897797158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:37,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/eb40a5f043a84411b83aeb95836b37d4 2024-12-11T06:15:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4984229695fb44efa7415835b619c3bf as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf 2024-12-11T06:15:37,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf, entries=250, sequenceid=489, filesize=16.8 K 2024-12-11T06:15:37,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/706c0e9a09994da083ea5d0919a9da8a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a 2024-12-11T06:15:37,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a, entries=150, sequenceid=489, filesize=12.0 K 2024-12-11T06:15:37,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/eb40a5f043a84411b83aeb95836b37d4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4 2024-12-11T06:15:37,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4, entries=150, sequenceid=489, filesize=12.0 K 2024-12-11T06:15:37,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=138.65 KB/141980 for 682402d56c3b62fdcea61b4be01f333c in 1291ms, sequenceid=489, compaction requested=true 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:37,239 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:37,239 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:37,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:37,240 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:37,240 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:37,240 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:37,240 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:37,240 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,240 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,240 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/708a7da8e6024ed399687a8a9bda56b5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=41.3 K 2024-12-11T06:15:37,240 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/9cc6ce8481804732813ad88d1785ee58, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.6 K 2024-12-11T06:15:37,241 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708a7da8e6024ed399687a8a9bda56b5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:37,241 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc6ce8481804732813ad88d1785ee58, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:37,241 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4bb8aad70a0422caffd47ce0f6a715b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733897734820 2024-12-11T06:15:37,241 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 162d3b8899b04235a79b40b04ab8a487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733897734820 2024-12-11T06:15:37,241 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4984229695fb44efa7415835b619c3bf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735940 2024-12-11T06:15:37,242 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 706c0e9a09994da083ea5d0919a9da8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735945 2024-12-11T06:15:37,255 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:37,255 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fdc47ded4897485f9a993495920a1ab7 is 50, key is test_row_0/B:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:37,258 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:37,258 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7550d2f103e84dec9f5c0b3112236abe is 50, key is test_row_0/A:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:37,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T06:15:37,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,261 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=138.65 KB heapSize=390.45 KB 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:37,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/d97940b603f04f0491194f8e14788735 is 50, key is test_row_0/A:col10/1733897736010/Put/seqid=0 2024-12-11T06:15:37,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742100_1276 (size=12949) 2024-12-11T06:15:37,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742101_1277 (size=12949) 2024-12-11T06:15:37,304 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7550d2f103e84dec9f5c0b3112236abe as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7550d2f103e84dec9f5c0b3112236abe 2024-12-11T06:15:37,312 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into 7550d2f103e84dec9f5c0b3112236abe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:37,312 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:37,312 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897737239; duration=0sec 2024-12-11T06:15:37,312 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:37,312 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:37,312 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:37,313 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:37,313 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:37,313 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,313 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/5d3b1accbd0646c9b1ecaa186d121769, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.6 K 2024-12-11T06:15:37,314 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3b1accbd0646c9b1ecaa186d121769, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733897734445 2024-12-11T06:15:37,315 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4adba295cb4949aaab35631a24c04fcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733897734820 2024-12-11T06:15:37,315 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb40a5f043a84411b83aeb95836b37d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735945 2024-12-11T06:15:37,321 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:37,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742102_1278 (size=12301) 2024-12-11T06:15:37,322 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/2b1408ad0fb54e54b5e4e5546bfe17c6 is 50, key is test_row_0/C:col10/1733897735945/Put/seqid=0 2024-12-11T06:15:37,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/d97940b603f04f0491194f8e14788735 2024-12-11T06:15:37,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/78a85d561865490490c451d3be30937f is 50, key is test_row_0/B:col10/1733897736010/Put/seqid=0 2024-12-11T06:15:37,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742103_1279 (size=12949) 2024-12-11T06:15:37,351 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/2b1408ad0fb54e54b5e4e5546bfe17c6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2b1408ad0fb54e54b5e4e5546bfe17c6 2024-12-11T06:15:37,357 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 2b1408ad0fb54e54b5e4e5546bfe17c6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:37,357 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:37,357 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897737239; duration=0sec 2024-12-11T06:15:37,358 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:37,358 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:37,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742104_1280 (size=12301) 2024-12-11T06:15:37,365 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/78a85d561865490490c451d3be30937f 2024-12-11T06:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/c76e90ec07c14af9bf601ff55e0bcdf6 is 50, key is test_row_0/C:col10/1733897736010/Put/seqid=0 2024-12-11T06:15:37,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742105_1281 (size=12301) 2024-12-11T06:15:37,708 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fdc47ded4897485f9a993495920a1ab7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fdc47ded4897485f9a993495920a1ab7 2024-12-11T06:15:37,715 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into fdc47ded4897485f9a993495920a1ab7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:37,715 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:37,715 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897737239; duration=0sec 2024-12-11T06:15:37,715 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:37,715 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:37,796 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/c76e90ec07c14af9bf601ff55e0bcdf6 2024-12-11T06:15:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/d97940b603f04f0491194f8e14788735 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735 2024-12-11T06:15:37,811 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:15:37,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/78a85d561865490490c451d3be30937f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f 2024-12-11T06:15:37,845 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:15:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/c76e90ec07c14af9bf601ff55e0bcdf6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6 2024-12-11T06:15:37,850 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:15:37,851 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~390.41 KB/399776, currentSize=0 B/0 for 682402d56c3b62fdcea61b4be01f333c in 591ms, sequenceid=525, compaction requested=false 2024-12-11T06:15:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-11T06:15:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-11T06:15:37,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-11T06:15:37,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9800 sec 2024-12-11T06:15:37,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9870 sec 2024-12-11T06:15:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T06:15:38,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:38,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-11T06:15:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:38,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:38,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:38,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:38,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:15:38,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:38,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4362000f02d3422fade003ad580e36a2 is 50, key is test_row_0/A:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:38,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897798181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897798181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897738179, max timestamp=1733897738184Num uniques 859; 2024-12-11T06:15:38,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897738179, max timestamp=1733897738184Num uniques 859; 2024-12-11T06:15:38,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897738179, max timestamp=1733897738184Num uniques 859; 2024-12-11T06:15:38,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:38,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742106_1282 (size=14741) 2024-12-11T06:15:38,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897798214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897798215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897798215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897798299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897798299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897798328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897798328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897798328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:38,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897798509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897798509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897798538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897798538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897798538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4362000f02d3422fade003ad580e36a2 2024-12-11T06:15:38,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4fd8ea59ce214db498ac810bac3c407e is 50, key is test_row_0/B:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:38,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742107_1283 (size=12301) 2024-12-11T06:15:38,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4fd8ea59ce214db498ac810bac3c407e 2024-12-11T06:15:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:38,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/8ad8830b847b46de88610f46b915cdb1 is 50, key is test_row_0/C:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:38,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742108_1284 (size=12301) 2024-12-11T06:15:38,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897798819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897798819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897798848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897798849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897798849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:38,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:38,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:38,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:38,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:38,959 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41397 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=4518703b2c1d,35425,1733897658824, table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-11T06:15:39,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/8ad8830b847b46de88610f46b915cdb1 2024-12-11T06:15:39,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/4362000f02d3422fade003ad580e36a2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2 2024-12-11T06:15:39,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:39,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2, entries=200, sequenceid=539, filesize=14.4 K 2024-12-11T06:15:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/4fd8ea59ce214db498ac810bac3c407e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e 2024-12-11T06:15:39,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e, entries=150, sequenceid=539, filesize=12.0 K 2024-12-11T06:15:39,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/8ad8830b847b46de88610f46b915cdb1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1 2024-12-11T06:15:39,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1, entries=150, sequenceid=539, filesize=12.0 K 2024-12-11T06:15:39,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 682402d56c3b62fdcea61b4be01f333c in 974ms, sequenceid=539, compaction requested=true 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:39,138 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:39,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:15:39,138 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:39,139 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:39,139 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:39,139 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,139 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fdc47ded4897485f9a993495920a1ab7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.7 K 2024-12-11T06:15:39,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:39,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:39,140 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,140 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7550d2f103e84dec9f5c0b3112236abe, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=39.1 K 2024-12-11T06:15:39,140 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc47ded4897485f9a993495920a1ab7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735945 2024-12-11T06:15:39,140 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7550d2f103e84dec9f5c0b3112236abe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735945 2024-12-11T06:15:39,141 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a85d561865490490c451d3be30937f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897736009 2024-12-11T06:15:39,142 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting d97940b603f04f0491194f8e14788735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897736009 2024-12-11T06:15:39,142 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd8ea59ce214db498ac810bac3c407e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:39,142 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4362000f02d3422fade003ad580e36a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:39,166 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:39,166 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/10b97ae91dab4538812d79f8ef7b8ead is 50, key is test_row_0/A:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:39,176 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:39,182 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2b3357ffc1a64bdfbcab3f1703d986eb is 50, key is test_row_0/B:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:39,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742109_1285 (size=13051) 2024-12-11T06:15:39,228 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/10b97ae91dab4538812d79f8ef7b8ead as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/10b97ae91dab4538812d79f8ef7b8ead 2024-12-11T06:15:39,237 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into 10b97ae91dab4538812d79f8ef7b8ead(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:39,237 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:39,237 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897739138; duration=0sec 2024-12-11T06:15:39,237 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:39,238 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:39,238 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:39,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742110_1286 (size=13051) 2024-12-11T06:15:39,242 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:39,243 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:39,243 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,243 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2b1408ad0fb54e54b5e4e5546bfe17c6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.7 K 2024-12-11T06:15:39,244 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1408ad0fb54e54b5e4e5546bfe17c6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1733897735945 2024-12-11T06:15:39,245 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c76e90ec07c14af9bf601ff55e0bcdf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897736009 2024-12-11T06:15:39,245 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad8830b847b46de88610f46b915cdb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:39,260 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:39,260 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/ce1c1750b7bf462798e580b3e998006b is 50, key is test_row_0/C:col10/1733897738159/Put/seqid=0 2024-12-11T06:15:39,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T06:15:39,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:39,266 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:39,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/54b7985061e44fc1a5117078ea733fb6 is 50, key is test_row_0/A:col10/1733897738213/Put/seqid=0 2024-12-11T06:15:39,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742111_1287 (size=13051) 2024-12-11T06:15:39,301 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/ce1c1750b7bf462798e580b3e998006b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ce1c1750b7bf462798e580b3e998006b 2024-12-11T06:15:39,309 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into ce1c1750b7bf462798e580b3e998006b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:39,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:39,309 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897739138; duration=0sec 2024-12-11T06:15:39,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:39,309 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:39,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742112_1288 (size=12301) 2024-12-11T06:15:39,338 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/54b7985061e44fc1a5117078ea733fb6 2024-12-11T06:15:39,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fd77acd07c3d4df68c8a17842ce7c1ae is 50, key is test_row_0/B:col10/1733897738213/Put/seqid=0 2024-12-11T06:15:39,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897799354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897799359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897799359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897799359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897799361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742113_1289 (size=12301) 2024-12-11T06:15:39,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897799468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897799479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,646 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/2b3357ffc1a64bdfbcab3f1703d986eb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2b3357ffc1a64bdfbcab3f1703d986eb 2024-12-11T06:15:39,651 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 2b3357ffc1a64bdfbcab3f1703d986eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:39,651 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:39,651 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897739138; duration=0sec 2024-12-11T06:15:39,651 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:39,651 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:39,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897799678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897799688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:39,791 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fd77acd07c3d4df68c8a17842ce7c1ae 2024-12-11T06:15:39,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d21092baf9cb49e39c0934003012b1a5 is 50, key is test_row_0/C:col10/1733897738213/Put/seqid=0 2024-12-11T06:15:39,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742114_1290 (size=12301) 2024-12-11T06:15:39,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897799988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897799999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:40,212 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d21092baf9cb49e39c0934003012b1a5 2024-12-11T06:15:40,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/54b7985061e44fc1a5117078ea733fb6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6 2024-12-11T06:15:40,226 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6, entries=150, sequenceid=577, filesize=12.0 K 2024-12-11T06:15:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/fd77acd07c3d4df68c8a17842ce7c1ae as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae 2024-12-11T06:15:40,232 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae, entries=150, sequenceid=577, filesize=12.0 K 2024-12-11T06:15:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d21092baf9cb49e39c0934003012b1a5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5 2024-12-11T06:15:40,239 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5, entries=150, sequenceid=577, filesize=12.0 K 2024-12-11T06:15:40,240 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 682402d56c3b62fdcea61b4be01f333c in 974ms, sequenceid=577, compaction requested=false 2024-12-11T06:15:40,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:40,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:40,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-11T06:15:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-11T06:15:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-11T06:15:40,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2110 sec 2024-12-11T06:15:40,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.2170 sec 2024-12-11T06:15:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:40,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:40,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/88d0cd5f45e249eb824d0a1b512f2187 is 50, key is test_row_0/A:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742115_1291 (size=17181) 2024-12-11T06:15:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897800416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897800416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897800422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897800498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897800509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897800529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897800529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897800529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897800738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897800738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897800738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:40,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/88d0cd5f45e249eb824d0a1b512f2187 2024-12-11T06:15:40,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/3e11530773214a6d8bf351fef295f4c1 is 50, key is test_row_0/B:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:40,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742116_1292 (size=12301) 2024-12-11T06:15:40,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/3e11530773214a6d8bf351fef295f4c1 2024-12-11T06:15:40,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/33b4bbd107af4bb5b270d40fc4ada0b4 is 50, key is test_row_0/C:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:40,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742117_1293 (size=12301) 2024-12-11T06:15:40,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/33b4bbd107af4bb5b270d40fc4ada0b4 2024-12-11T06:15:40,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/88d0cd5f45e249eb824d0a1b512f2187 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187 2024-12-11T06:15:40,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187, entries=250, sequenceid=593, filesize=16.8 K 2024-12-11T06:15:40,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/3e11530773214a6d8bf351fef295f4c1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1 2024-12-11T06:15:40,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1, entries=150, sequenceid=593, filesize=12.0 K 2024-12-11T06:15:40,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/33b4bbd107af4bb5b270d40fc4ada0b4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4 2024-12-11T06:15:40,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4, entries=150, sequenceid=593, filesize=12.0 K 2024-12-11T06:15:40,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 682402d56c3b62fdcea61b4be01f333c in 541ms, sequenceid=593, compaction requested=true 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:40,922 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:40,922 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:40,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:40,923 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:40,923 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:40,923 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:40,923 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2b3357ffc1a64bdfbcab3f1703d986eb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.8 K 2024-12-11T06:15:40,923 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:40,923 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:40,923 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:40,923 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/10b97ae91dab4538812d79f8ef7b8ead, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=41.5 K 2024-12-11T06:15:40,924 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3357ffc1a64bdfbcab3f1703d986eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:40,924 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10b97ae91dab4538812d79f8ef7b8ead, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:40,924 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting fd77acd07c3d4df68c8a17842ce7c1ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733897738179 2024-12-11T06:15:40,925 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b7985061e44fc1a5117078ea733fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733897738179 2024-12-11T06:15:40,925 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e11530773214a6d8bf351fef295f4c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:40,925 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88d0cd5f45e249eb824d0a1b512f2187, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:40,934 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:40,935 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:40,935 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/80f726bd192d4d5db691c49acc08ea93 is 50, key is test_row_0/B:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:40,935 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/d33ed457b2b4459fbad3d86da3cd7f23 is 50, key is test_row_0/A:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:40,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742118_1294 (size=13153) 2024-12-11T06:15:40,973 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/80f726bd192d4d5db691c49acc08ea93 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/80f726bd192d4d5db691c49acc08ea93 2024-12-11T06:15:40,981 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 80f726bd192d4d5db691c49acc08ea93(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:40,981 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:40,981 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897740922; duration=0sec 2024-12-11T06:15:40,981 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:40,981 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:40,981 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:40,982 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:40,982 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:40,982 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:40,983 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ce1c1750b7bf462798e580b3e998006b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.8 K 2024-12-11T06:15:40,983 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1c1750b7bf462798e580b3e998006b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733897738159 2024-12-11T06:15:40,983 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d21092baf9cb49e39c0934003012b1a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1733897738179 2024-12-11T06:15:40,984 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b4bbd107af4bb5b270d40fc4ada0b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742119_1295 (size=13153) 2024-12-11T06:15:40,999 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:40,999 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/64dd2d3742084f84ab7c040932813390 is 50, key is test_row_0/C:col10/1733897740380/Put/seqid=0 2024-12-11T06:15:41,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742120_1296 (size=13153) 2024-12-11T06:15:41,044 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/64dd2d3742084f84ab7c040932813390 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64dd2d3742084f84ab7c040932813390 2024-12-11T06:15:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:41,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733897740405, max timestamp=1733897741050Num uniques 859; 2024-12-11T06:15:41,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897740405, max timestamp=1733897741050Num uniques 859; 2024-12-11T06:15:41,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:41,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:41,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897740405, max timestamp=1733897741050Num uniques 859; 2024-12-11T06:15:41,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:15:41,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=33.54 KB heapSize=81.47 KB 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:41,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/35aae4292f4d447485267a846eae4688 is 50, key is test_row_0/A:col10/1733897740405/Put/seqid=0 2024-12-11T06:15:41,066 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 64dd2d3742084f84ab7c040932813390(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:41,066 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:41,066 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897740922; duration=0sec 2024-12-11T06:15:41,067 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:41,068 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:41,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742121_1297 (size=12301) 2024-12-11T06:15:41,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897741088, max timestamp=1733897741101Num uniques 859; 2024-12-11T06:15:41,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:41,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897741088, max timestamp=1733897741104Num uniques 859; 2024-12-11T06:15:41,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:41,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897741088, max timestamp=1733897741104Num uniques 859; 2024-12-11T06:15:41,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7686, totalHeapSize=17712, min timestamp=1733897741127, max timestamp=1733897741139Num uniques 1018; 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46616, new segment data size=7686, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1733897741127, max timestamp=1733897741139Num uniques 1018; 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7686, totalHeapSize=17712, min timestamp=1733897741127, max timestamp=1733897741139Num uniques 1018; 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=7680, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:15:41,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46616, new segment data size=7686, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T06:15:41,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897741180, max timestamp=1733897741196Num uniques 1027; 2024-12-11T06:15:41,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897741180, max timestamp=1733897741196Num uniques 1027; 2024-12-11T06:15:41,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:41,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:41,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897741180, max timestamp=1733897741196Num uniques 1027; 2024-12-11T06:15:41,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T06:15:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733897741223, max timestamp=1733897741240Num uniques 1036; 2024-12-11T06:15:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733897741223, max timestamp=1733897741240Num uniques 1036; 2024-12-11T06:15:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=10796, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-11T06:15:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=10796, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-11T06:15:41,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897741223, max timestamp=1733897741240Num uniques 1036; 2024-12-11T06:15:41,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733897741240, max timestamp=1733897741291Num uniques 1009; 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733897741240, max timestamp=1733897741291Num uniques 1095; 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733897741240, max timestamp=1733897741291Num uniques 1095; 2024-12-11T06:15:41,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-11T06:15:41,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897801326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897801328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897741323, max timestamp=1733897741330Num uniques 1104; 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897741323, max timestamp=1733897741330Num uniques 1104; 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897741323, max timestamp=1733897741330Num uniques 1104; 2024-12-11T06:15:41,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-11T06:15:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,396 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/d33ed457b2b4459fbad3d86da3cd7f23 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d33ed457b2b4459fbad3d86da3cd7f23 2024-12-11T06:15:41,401 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into d33ed457b2b4459fbad3d86da3cd7f23(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:41,401 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:41,401 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897740922; duration=0sec 2024-12-11T06:15:41,401 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:41,401 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733897741360, max timestamp=1733897741412Num uniques 1013; 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733897741360, max timestamp=1733897741412Num uniques 1013; 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733897741360, max timestamp=1733897741412Num uniques 1013; 2024-12-11T06:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733897741447, max timestamp=1733897741464Num uniques 1022; 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733897741447, max timestamp=1733897741464Num uniques 1022; 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733897741447, max timestamp=1733897741464Num uniques 1022; 2024-12-11T06:15:41,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-12-11T06:15:41,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/35aae4292f4d447485267a846eae4688 2024-12-11T06:15:41,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1716ec5ace0441b993428ec8e0139a09 is 50, key is test_row_0/B:col10/1733897740405/Put/seqid=0 2024-12-11T06:15:41,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742122_1298 (size=12301) 2024-12-11T06:15:41,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733897741486, max timestamp=1733897741506Num uniques 1081; 2024-12-11T06:15:41,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733897741486, max timestamp=1733897741506Num uniques 1081; 2024-12-11T06:15:41,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-11T06:15:41,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-11T06:15:41,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733897741486, max timestamp=1733897741506Num uniques 1081; 2024-12-11T06:15:41,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733897741517, max timestamp=1733897741530Num uniques 1040; 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733897741517, max timestamp=1733897741530Num uniques 1040; 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733897741517, max timestamp=1733897741530Num uniques 1040; 2024-12-11T06:15:41,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733897741545, max timestamp=1733897741554Num uniques 1099; 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1733897741545, max timestamp=1733897741554Num uniques 1099; 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733897741548, max timestamp=1733897741554Num uniques 1099; 2024-12-11T06:15:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-11T06:15:41,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897801575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897801576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1733897741574, max timestamp=1733897741581Num uniques 1208; 2024-12-11T06:15:41,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=9518, suffix heap size=153600,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=208 2024-12-11T06:15:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11808, totalHeapSize=27072, min timestamp=1733897741572, max timestamp=1733897741581Num uniques 1208; 2024-12-11T06:15:41,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=11808, suffix heap size=153600,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=258 2024-12-11T06:15:41,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897741574, max timestamp=1733897741581Num uniques 1108; 2024-12-11T06:15:41,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=6870, suffix heap size=143200,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=150 2024-12-11T06:15:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733897741614, max timestamp=1733897741645Num uniques 1067; 2024-12-11T06:15:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-11T06:15:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733897741614, max timestamp=1733897741645Num uniques 1117; 2024-12-11T06:15:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-12-11T06:15:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733897741614, max timestamp=1733897741645Num uniques 1009; 2024-12-11T06:15:41,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=9932, suffix heap size=132904,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=217 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733897741667, max timestamp=1733897741673Num uniques 1076; 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733897741667, max timestamp=1733897741673Num uniques 1076; 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733897741667, max timestamp=1733897741673Num uniques 1076; 2024-12-11T06:15:41,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-12-11T06:15:41,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733897741692, max timestamp=1733897741699Num uniques 1085; 2024-12-11T06:15:41,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-12-11T06:15:41,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733897741692, max timestamp=1733897741699Num uniques 1085; 2024-12-11T06:15:41,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-11T06:15:41,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733897741692, max timestamp=1733897741699Num uniques 1085; 2024-12-11T06:15:41,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-12-11T06:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733897741714, max timestamp=1733897741726Num uniques 1144; 2024-12-11T06:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-12-11T06:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733897741714, max timestamp=1733897741726Num uniques 1144; 2024-12-11T06:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-12-11T06:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733897741714, max timestamp=1733897741726Num uniques 1044; 2024-12-11T06:15:41,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=13464, suffix heap size=136544,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=294 2024-12-11T06:15:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897801747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897801747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897801748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897801749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897801749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897801858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897801858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897801858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897801858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897801859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:41,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1716ec5ace0441b993428ec8e0139a09 2024-12-11T06:15:41,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/28e09095661049f69b61b20d1d2efd46 is 50, key is test_row_0/C:col10/1733897740405/Put/seqid=0 2024-12-11T06:15:41,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742123_1299 (size=12301) 2024-12-11T06:15:41,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/28e09095661049f69b61b20d1d2efd46 2024-12-11T06:15:41,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/35aae4292f4d447485267a846eae4688 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688 2024-12-11T06:15:41,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688, entries=150, sequenceid=615, filesize=12.0 K 2024-12-11T06:15:41,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1716ec5ace0441b993428ec8e0139a09 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09 2024-12-11T06:15:41,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09, entries=150, sequenceid=615, filesize=12.0 K 2024-12-11T06:15:41,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/28e09095661049f69b61b20d1d2efd46 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46 2024-12-11T06:15:41,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46, entries=150, sequenceid=615, filesize=12.0 K 2024-12-11T06:15:41,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=154.31 KB/158010 for 682402d56c3b62fdcea61b4be01f333c in 864ms, sequenceid=615, compaction requested=false 2024-12-11T06:15:41,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:42,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=161.02 KB heapSize=452.72 KB 2024-12-11T06:15:42,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733897741740, max timestamp=1733897742071Num uniques 1153; 2024-12-11T06:15:42,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-12-11T06:15:42,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733897741740, max timestamp=1733897742071Num uniques 1153; 2024-12-11T06:15:42,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-12-11T06:15:42,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733897741740, max timestamp=1733897742071Num uniques 1153; 2024-12-11T06:15:42,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=7005, suffix heap size=147880,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=153 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:42,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b696789eb92a4669b5eb77c36f6da5a9 is 50, key is test_row_0/A:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742124_1300 (size=12301) 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897742091, max timestamp=1733897742101Num uniques 859; 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897742091, max timestamp=1733897742101Num uniques 859; 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897742091, max timestamp=1733897742101Num uniques 859; 2024-12-11T06:15:42,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897802118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897802118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733897742115, max timestamp=1733897742123Num uniques 1018; 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733897742115, max timestamp=1733897742123Num uniques 1018; 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733897742115, max timestamp=1733897742123Num uniques 1018; 2024-12-11T06:15:42,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-11T06:15:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897802156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897802158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897802161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T06:15:42,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:42,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-11T06:15:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:42,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:42,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:42,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:42,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897802229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897802229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897802268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897802268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897802268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:42,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:42,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:42,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897802439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897802440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897802478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897802480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897802481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=915 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b696789eb92a4669b5eb77c36f6da5a9 2024-12-11T06:15:42,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/7b689b8e54ea4263bb6678c64063ee4d is 50, key is test_row_0/B:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:42,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742125_1301 (size=12301) 2024-12-11T06:15:42,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897802749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897802761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897802793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897802801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897802801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:42,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:42,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:42,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=915 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/7b689b8e54ea4263bb6678c64063ee4d 2024-12-11T06:15:42,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b9c617cb7d6949c3bbb813cb929c781d is 50, key is test_row_0/C:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:42,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742126_1302 (size=12301) 2024-12-11T06:15:42,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=915 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b9c617cb7d6949c3bbb813cb929c781d 2024-12-11T06:15:42,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:42,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:42,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/b696789eb92a4669b5eb77c36f6da5a9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9 2024-12-11T06:15:43,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9, entries=150, sequenceid=915, filesize=12.0 K 2024-12-11T06:15:43,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/7b689b8e54ea4263bb6678c64063ee4d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d 2024-12-11T06:15:43,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d, entries=150, sequenceid=915, filesize=12.0 K 2024-12-11T06:15:43,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/b9c617cb7d6949c3bbb813cb929c781d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d 2024-12-11T06:15:43,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d, entries=150, sequenceid=915, filesize=12.0 K 2024-12-11T06:15:43,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.59 KB/109152, currentSize=147.60 KB/151140 for 682402d56c3b62fdcea61b4be01f333c in 960ms, sequenceid=915, compaction requested=true 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:43,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T06:15:43,032 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:43,034 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:43,035 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:43,035 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:43,035 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:43,035 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d33ed457b2b4459fbad3d86da3cd7f23, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.9 K 2024-12-11T06:15:43,036 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d33ed457b2b4459fbad3d86da3cd7f23, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:43,036 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 35aae4292f4d447485267a846eae4688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733897740405 2024-12-11T06:15:43,037 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b696789eb92a4669b5eb77c36f6da5a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:43,038 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:43,038 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:43,038 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:43,038 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64dd2d3742084f84ab7c040932813390, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.9 K 2024-12-11T06:15:43,041 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64dd2d3742084f84ab7c040932813390, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:43,042 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e09095661049f69b61b20d1d2efd46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733897740405 2024-12-11T06:15:43,043 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c617cb7d6949c3bbb813cb929c781d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:43,070 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:43,070 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/bf74b60dcb274bfca68ed5c096184a2a is 50, key is test_row_0/A:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:43,087 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:43,088 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/560a28f04ca64095a3454887d93dcc12 is 50, key is test_row_0/C:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:43,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742127_1303 (size=13255) 2024-12-11T06:15:43,119 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/bf74b60dcb274bfca68ed5c096184a2a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/bf74b60dcb274bfca68ed5c096184a2a 2024-12-11T06:15:43,127 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into bf74b60dcb274bfca68ed5c096184a2a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:43,127 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:43,127 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897743032; duration=0sec 2024-12-11T06:15:43,127 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:43,127 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:43,127 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:43,130 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:43,130 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:43,130 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:43,130 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/80f726bd192d4d5db691c49acc08ea93, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=36.9 K 2024-12-11T06:15:43,131 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 80f726bd192d4d5db691c49acc08ea93, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1733897739352 2024-12-11T06:15:43,132 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1716ec5ace0441b993428ec8e0139a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733897740405 2024-12-11T06:15:43,132 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b689b8e54ea4263bb6678c64063ee4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742128_1304 (size=13255) 2024-12-11T06:15:43,148 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/560a28f04ca64095a3454887d93dcc12 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/560a28f04ca64095a3454887d93dcc12 2024-12-11T06:15:43,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T06:15:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:43,154 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-12-11T06:15:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:43,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:43,155 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#263 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:43,156 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 560a28f04ca64095a3454887d93dcc12(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:43,156 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:43,156 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897743032; duration=0sec 2024-12-11T06:15:43,156 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/0d12d34f60fb48c981e692519db605f0 is 50, key is test_row_0/B:col10/1733897742071/Put/seqid=0 2024-12-11T06:15:43,156 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:43,156 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84c126f6029d4ac9a3694a7fdcc469b9 is 50, key is test_row_0/A:col10/1733897742155/Put/seqid=0 2024-12-11T06:15:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742130_1306 (size=12301) 2024-12-11T06:15:43,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742129_1305 (size=13255) 2024-12-11T06:15:43,230 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/0d12d34f60fb48c981e692519db605f0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/0d12d34f60fb48c981e692519db605f0 2024-12-11T06:15:43,251 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into 0d12d34f60fb48c981e692519db605f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:43,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:43,251 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897743032; duration=0sec 2024-12-11T06:15:43,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:43,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:43,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:43,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897803289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897803293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897803309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897803318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897803318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897803399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897803417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,606 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=970 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84c126f6029d4ac9a3694a7fdcc469b9 2024-12-11T06:15:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897803613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/253c07650b144e31986b7a917eb5cacb is 50, key is test_row_0/B:col10/1733897742155/Put/seqid=0 2024-12-11T06:15:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897803629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742131_1307 (size=12301) 2024-12-11T06:15:43,678 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=970 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/253c07650b144e31986b7a917eb5cacb 2024-12-11T06:15:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d581ac006a7e49b0bb9dbdc6913ef1d1 is 50, key is test_row_0/C:col10/1733897742155/Put/seqid=0 2024-12-11T06:15:43,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742132_1308 (size=12301) 2024-12-11T06:15:43,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897803919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897803939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=970 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d581ac006a7e49b0bb9dbdc6913ef1d1 2024-12-11T06:15:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/84c126f6029d4ac9a3694a7fdcc469b9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9 2024-12-11T06:15:44,200 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9, entries=150, sequenceid=970, filesize=12.0 K 2024-12-11T06:15:44,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/253c07650b144e31986b7a917eb5cacb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb 2024-12-11T06:15:44,214 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb, entries=150, sequenceid=970, filesize=12.0 K 2024-12-11T06:15:44,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/d581ac006a7e49b0bb9dbdc6913ef1d1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1 2024-12-11T06:15:44,243 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1, entries=150, sequenceid=970, filesize=12.0 K 2024-12-11T06:15:44,252 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for 682402d56c3b62fdcea61b4be01f333c in 1098ms, sequenceid=970, compaction requested=false 2024-12-11T06:15:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-11T06:15:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-11T06:15:44,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-11T06:15:44,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0670 sec 2024-12-11T06:15:44,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0820 sec 2024-12-11T06:15:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:44,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:44,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T06:15:44,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:44,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-11T06:15:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:44,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:44,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:44,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:44,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0330a39e88ee45b1938de1eef9425ca3 is 50, key is test_row_0/A:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742133_1309 (size=12301) 2024-12-11T06:15:44,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=982 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0330a39e88ee45b1938de1eef9425ca3 2024-12-11T06:15:44,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897744388, max timestamp=1733897744417Num uniques 859; 2024-12-11T06:15:44,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:44,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897744388, max timestamp=1733897744417Num uniques 859; 2024-12-11T06:15:44,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897744388, max timestamp=1733897744417Num uniques 859; 2024-12-11T06:15:44,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:44,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:44,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/60ffbfbfde0d412ca50ed28d3315f329 is 50, key is test_row_0/B:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:44,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742134_1310 (size=12301) 2024-12-11T06:15:44,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=982 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/60ffbfbfde0d412ca50ed28d3315f329 2024-12-11T06:15:44,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897744460, max timestamp=1733897744494Num uniques 1018; 2024-12-11T06:15:44,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:15:44,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897744460, max timestamp=1733897744494Num uniques 1018; 2024-12-11T06:15:44,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:15:44,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/6c79a3b5e199483c964834122ec2ac1a is 50, key is test_row_0/C:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:44,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897744469, max timestamp=1733897744494Num uniques 1018; 2024-12-11T06:15:44,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-11T06:15:44,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T06:15:44,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742135_1311 (size=12301) 2024-12-11T06:15:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897804543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897804548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897804554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897804558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897804558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:44,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897804661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T06:15:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897804670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897804670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897804670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897804677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T06:15:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53184 deadline: 1733897804878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53138 deadline: 1733897804878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53168 deadline: 1733897804881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53132 deadline: 1733897804878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53154 deadline: 1733897804889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:15:44,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=982 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/6c79a3b5e199483c964834122ec2ac1a 2024-12-11T06:15:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:44,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T06:15:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:44,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/0330a39e88ee45b1938de1eef9425ca3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3 2024-12-11T06:15:45,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3, entries=150, sequenceid=982, filesize=12.0 K 2024-12-11T06:15:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/60ffbfbfde0d412ca50ed28d3315f329 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329 2024-12-11T06:15:45,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329, entries=150, sequenceid=982, filesize=12.0 K 2024-12-11T06:15:45,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/6c79a3b5e199483c964834122ec2ac1a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a 2024-12-11T06:15:45,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a, entries=150, sequenceid=982, filesize=12.0 K 2024-12-11T06:15:45,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.85 KB/141162 for 682402d56c3b62fdcea61b4be01f333c in 767ms, sequenceid=982, compaction requested=true 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 682402d56c3b62fdcea61b4be01f333c:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:45,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T06:15:45,109 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:45,109 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:45,117 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:45,117 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/A is initiating minor compaction (all files) 2024-12-11T06:15:45,117 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/A in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:45,117 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/bf74b60dcb274bfca68ed5c096184a2a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=37.0 K 2024-12-11T06:15:45,117 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:45,117 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/C is initiating minor compaction (all files) 2024-12-11T06:15:45,118 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/C in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:45,118 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/560a28f04ca64095a3454887d93dcc12, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=37.0 K 2024-12-11T06:15:45,118 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 560a28f04ca64095a3454887d93dcc12, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:45,118 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf74b60dcb274bfca68ed5c096184a2a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:45,119 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c126f6029d4ac9a3694a7fdcc469b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=970, earliestPutTs=1733897742153 2024-12-11T06:15:45,119 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d581ac006a7e49b0bb9dbdc6913ef1d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=970, earliestPutTs=1733897742153 2024-12-11T06:15:45,120 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0330a39e88ee45b1938de1eef9425ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=982, earliestPutTs=1733897743274 2024-12-11T06:15:45,120 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c79a3b5e199483c964834122ec2ac1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=982, earliestPutTs=1733897743274 2024-12-11T06:15:45,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T06:15:45,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:45,153 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=137.85 KB heapSize=392.47 KB 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:45,155 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:45,161 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:45,161 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/62a92fcf5fbc48028c5319553fe5479f is 50, key is test_row_0/C:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:45,164 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-12-11T06:15:45,164 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,164 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,164 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,164 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-12-11T06:15:45,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,164 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,164 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,165 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,165 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-12-11T06:15:45,165 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,166 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,166 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,167 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/e6ff1ee4e8674bddb4a4e08288462718 is 50, key is test_row_0/A:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:45,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c17ef9205ea54d39ab34daf6bf8e435e is 50, key is test_row_0/A:col10/1733897744552/Put/seqid=0 2024-12-11T06:15:45,171 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-12-11T06:15:45,171 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,171 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,171 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,174 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-12-11T06:15:45,174 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,174 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,174 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742138_1314 (size=12301) 2024-12-11T06:15:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742136_1312 (size=13357) 2024-12-11T06:15:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742137_1313 (size=13357) 2024-12-11T06:15:45,192 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/62a92fcf5fbc48028c5319553fe5479f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/62a92fcf5fbc48028c5319553fe5479f 2024-12-11T06:15:45,195 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/e6ff1ee4e8674bddb4a4e08288462718 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e6ff1ee4e8674bddb4a4e08288462718 2024-12-11T06:15:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:45,202 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/C of 682402d56c3b62fdcea61b4be01f333c into 62a92fcf5fbc48028c5319553fe5479f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:45,202 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:45,202 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/A of 682402d56c3b62fdcea61b4be01f333c into e6ff1ee4e8674bddb4a4e08288462718(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:45,202 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-12-11T06:15:45,202 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:45,202 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/A, priority=13, startTime=1733897745107; duration=0sec 2024-12-11T06:15:45,202 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,202 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,202 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,202 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,202 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:45,202 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/C, priority=13, startTime=1733897745107; duration=0sec 2024-12-11T06:15:45,203 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:45,203 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:C 2024-12-11T06:15:45,203 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:A 2024-12-11T06:15:45,203 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. as already flushing 2024-12-11T06:15:45,203 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-12-11T06:15:45,203 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,203 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,204 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,205 INFO [Thread-1036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1036. 2024-12-11T06:15:45,205 DEBUG [Thread-1036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,205 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,205 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,208 INFO [Thread-1034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1034. 2024-12-11T06:15:45,209 DEBUG [Thread-1034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,209 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,209 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,211 INFO [Thread-1032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1032. 2024-12-11T06:15:45,211 DEBUG [Thread-1032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:15:45,211 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,211 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:45,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:45,217 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:45,217 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 682402d56c3b62fdcea61b4be01f333c/B is initiating minor compaction (all files) 2024-12-11T06:15:45,217 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 682402d56c3b62fdcea61b4be01f333c/B in TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:45,217 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/0d12d34f60fb48c981e692519db605f0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp, totalSize=37.0 K 2024-12-11T06:15:45,218 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d12d34f60fb48c981e692519db605f0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=915, earliestPutTs=1733897742071 2024-12-11T06:15:45,218 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 253c07650b144e31986b7a917eb5cacb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=970, earliestPutTs=1733897742153 2024-12-11T06:15:45,218 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ffbfbfde0d412ca50ed28d3315f329, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=982, earliestPutTs=1733897743274 2024-12-11T06:15:45,225 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 682402d56c3b62fdcea61b4be01f333c#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:45,225 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/d1c81da90234456a9d16686004df0d19 is 50, key is test_row_0/B:col10/1733897743281/Put/seqid=0 2024-12-11T06:15:45,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742139_1315 (size=13357) 2024-12-11T06:15:45,247 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/d1c81da90234456a9d16686004df0d19 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/d1c81da90234456a9d16686004df0d19 2024-12-11T06:15:45,261 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 682402d56c3b62fdcea61b4be01f333c/B of 682402d56c3b62fdcea61b4be01f333c into d1c81da90234456a9d16686004df0d19(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:45,261 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:45,261 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., storeName=682402d56c3b62fdcea61b4be01f333c/B, priority=13, startTime=1733897745107; duration=0sec 2024-12-11T06:15:45,261 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:45,261 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 682402d56c3b62fdcea61b4be01f333c:B 2024-12-11T06:15:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:45,587 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1036 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c17ef9205ea54d39ab34daf6bf8e435e 2024-12-11T06:15:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1d09cecf00a24823ab03830153ca4515 is 50, key is test_row_0/B:col10/1733897744552/Put/seqid=0 2024-12-11T06:15:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742140_1316 (size=12301) 2024-12-11T06:15:46,005 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1036 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1d09cecf00a24823ab03830153ca4515 2024-12-11T06:15:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3e1594cdadaf4c4cb81bd4ec9652658e is 50, key is test_row_0/C:col10/1733897744552/Put/seqid=0 2024-12-11T06:15:46,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742141_1317 (size=12301) 2024-12-11T06:15:46,427 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=1036 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3e1594cdadaf4c4cb81bd4ec9652658e 2024-12-11T06:15:46,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/c17ef9205ea54d39ab34daf6bf8e435e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c17ef9205ea54d39ab34daf6bf8e435e 2024-12-11T06:15:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c17ef9205ea54d39ab34daf6bf8e435e, entries=150, sequenceid=1036, filesize=12.0 K 2024-12-11T06:15:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/1d09cecf00a24823ab03830153ca4515 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1d09cecf00a24823ab03830153ca4515 2024-12-11T06:15:46,446 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1d09cecf00a24823ab03830153ca4515, entries=150, sequenceid=1036, filesize=12.0 K 2024-12-11T06:15:46,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/3e1594cdadaf4c4cb81bd4ec9652658e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3e1594cdadaf4c4cb81bd4ec9652658e 2024-12-11T06:15:46,452 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3e1594cdadaf4c4cb81bd4ec9652658e, entries=150, sequenceid=1036, filesize=12.0 K 2024-12-11T06:15:46,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~137.85 KB/141162, heapSize ~392.42 KB/401840, currentSize=33.54 KB/34350 for 682402d56c3b62fdcea61b4be01f333c in 1300ms, sequenceid=1036, compaction requested=false 2024-12-11T06:15:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 682402d56c3b62fdcea61b4be01f333c: 2024-12-11T06:15:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:46,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-11T06:15:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-11T06:15:46,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-11T06:15:46,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-12-11T06:15:46,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1340 sec 2024-12-11T06:15:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T06:15:46,488 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 248 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 194 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 169 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3295 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3367 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3290 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3263 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3341 2024-12-11T06:15:46,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:15:46,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:15:46,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:15:46,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:46,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:46,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:15:46,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:15:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:46,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897746496"}]},"ts":"1733897746496"} 2024-12-11T06:15:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:46,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:15:46,504 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:15:46,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:15:46,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, UNASSIGN}] 2024-12-11T06:15:46,514 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, UNASSIGN 2024-12-11T06:15:46,515 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=682402d56c3b62fdcea61b4be01f333c, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:46,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, UNASSIGN because future has completed 2024-12-11T06:15:46,521 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:15:46,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:46,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T06:15:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:46,673 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:46,673 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:15:46,673 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 682402d56c3b62fdcea61b4be01f333c, disabling compactions & flushes 2024-12-11T06:15:46,673 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:46,673 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:46,673 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. after waiting 0 ms 2024-12-11T06:15:46,673 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:46,674 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 682402d56c3b62fdcea61b4be01f333c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=A 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=B 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 682402d56c3b62fdcea61b4be01f333c, store=C 2024-12-11T06:15:46,674 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:46,677 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7658a24e84274e8c9d3b16931a20d0c3 is 50, key is test_row_0/A:col10/1733897745210/Put/seqid=0 2024-12-11T06:15:46,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742142_1318 (size=12301) 2024-12-11T06:15:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:47,081 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1047 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7658a24e84274e8c9d3b16931a20d0c3 2024-12-11T06:15:47,089 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/cc0dcb783e0c4d8f9d6daefe56bee16b is 50, key is test_row_0/B:col10/1733897745210/Put/seqid=0 2024-12-11T06:15:47,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742143_1319 (size=12301) 2024-12-11T06:15:47,094 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1047 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/cc0dcb783e0c4d8f9d6daefe56bee16b 2024-12-11T06:15:47,100 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f24a1805a6534b53893394dd5e7e96b5 is 50, key is test_row_0/C:col10/1733897745210/Put/seqid=0 2024-12-11T06:15:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742144_1320 (size=12301) 2024-12-11T06:15:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:47,515 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1047 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f24a1805a6534b53893394dd5e7e96b5 2024-12-11T06:15:47,519 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/A/7658a24e84274e8c9d3b16931a20d0c3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7658a24e84274e8c9d3b16931a20d0c3 2024-12-11T06:15:47,524 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7658a24e84274e8c9d3b16931a20d0c3, entries=150, sequenceid=1047, filesize=12.0 K 2024-12-11T06:15:47,525 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/B/cc0dcb783e0c4d8f9d6daefe56bee16b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/cc0dcb783e0c4d8f9d6daefe56bee16b 2024-12-11T06:15:47,530 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/cc0dcb783e0c4d8f9d6daefe56bee16b, entries=150, sequenceid=1047, filesize=12.0 K 2024-12-11T06:15:47,531 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/.tmp/C/f24a1805a6534b53893394dd5e7e96b5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f24a1805a6534b53893394dd5e7e96b5 2024-12-11T06:15:47,536 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f24a1805a6534b53893394dd5e7e96b5, entries=150, sequenceid=1047, filesize=12.0 K 2024-12-11T06:15:47,537 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 682402d56c3b62fdcea61b4be01f333c in 864ms, sequenceid=1047, compaction requested=true 2024-12-11T06:15:47,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/ce7b873f5d3847f69e2a48e91cdb8855, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e38e7e116954a5a8f868a7f46b24ef5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b3f91397ba3943b087913bdc1f419544, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/a7d305a55dc14b949f13770d647b0f4c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/708a7da8e6024ed399687a8a9bda56b5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7550d2f103e84dec9f5c0b3112236abe, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/10b97ae91dab4538812d79f8ef7b8ead, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d33ed457b2b4459fbad3d86da3cd7f23, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/bf74b60dcb274bfca68ed5c096184a2a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3] to archive 2024-12-11T06:15:47,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:47,544 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84254f7ab54c4ed49cdfc761d9a2f579 2024-12-11T06:15:47,544 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e38e7e116954a5a8f868a7f46b24ef5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e38e7e116954a5a8f868a7f46b24ef5 2024-12-11T06:15:47,544 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c35ad807132549cbac938e6dee0a4757 2024-12-11T06:15:47,544 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/ce7b873f5d3847f69e2a48e91cdb8855 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/ce7b873f5d3847f69e2a48e91cdb8855 2024-12-11T06:15:47,545 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b279704365204713b9d602415ff7de53 2024-12-11T06:15:47,545 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/6e8dcac835694b26a555142b9d3c2d28 2024-12-11T06:15:47,545 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7aa812dfeed44a96be6b40fd001a2448 2024-12-11T06:15:47,547 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0ec078b5d431490a850d7ed43018c064 2024-12-11T06:15:47,551 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b24179c634d546b09ba3b029e59125ee 2024-12-11T06:15:47,551 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/07e23f936f67473595ead9bdc03d51a0 2024-12-11T06:15:47,551 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b3f91397ba3943b087913bdc1f419544 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b3f91397ba3943b087913bdc1f419544 2024-12-11T06:15:47,551 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c1314643c09642548367e686bf21b248 2024-12-11T06:15:47,552 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e3afb47d7bf4492d89763281768fb345 2024-12-11T06:15:47,552 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/a7d305a55dc14b949f13770d647b0f4c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/a7d305a55dc14b949f13770d647b0f4c 2024-12-11T06:15:47,552 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/23ce0c00f8574f0db396c0e7b1861878 2024-12-11T06:15:47,557 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b4bb8aad70a0422caffd47ce0f6a715b 2024-12-11T06:15:47,563 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/08211b4df9514effa32b940872cfd59d 2024-12-11T06:15:47,564 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/10b97ae91dab4538812d79f8ef7b8ead to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/10b97ae91dab4538812d79f8ef7b8ead 2024-12-11T06:15:47,564 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7550d2f103e84dec9f5c0b3112236abe to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7550d2f103e84dec9f5c0b3112236abe 2024-12-11T06:15:47,564 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4362000f02d3422fade003ad580e36a2 2024-12-11T06:15:47,564 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/708a7da8e6024ed399687a8a9bda56b5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/708a7da8e6024ed399687a8a9bda56b5 2024-12-11T06:15:47,565 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d97940b603f04f0491194f8e14788735 2024-12-11T06:15:47,565 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/4984229695fb44efa7415835b619c3bf 2024-12-11T06:15:47,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/54b7985061e44fc1a5117078ea733fb6 2024-12-11T06:15:47,569 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/88d0cd5f45e249eb824d0a1b512f2187 2024-12-11T06:15:47,571 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/35aae4292f4d447485267a846eae4688 2024-12-11T06:15:47,571 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/bf74b60dcb274bfca68ed5c096184a2a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/bf74b60dcb274bfca68ed5c096184a2a 2024-12-11T06:15:47,572 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/84c126f6029d4ac9a3694a7fdcc469b9 2024-12-11T06:15:47,572 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/0330a39e88ee45b1938de1eef9425ca3 2024-12-11T06:15:47,572 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d33ed457b2b4459fbad3d86da3cd7f23 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/d33ed457b2b4459fbad3d86da3cd7f23 2024-12-11T06:15:47,576 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/b696789eb92a4669b5eb77c36f6da5a9 2024-12-11T06:15:47,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/b0c404f0a7f74e45a5081e2c004bfea4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2ba98f02ae34429dbecbd2465499ff63, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/09112a70726647a99e7b317360d593cb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/f9a634e32837464aa02e980889b720cf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/9cc6ce8481804732813ad88d1785ee58, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fdc47ded4897485f9a993495920a1ab7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2b3357ffc1a64bdfbcab3f1703d986eb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/80f726bd192d4d5db691c49acc08ea93, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/0d12d34f60fb48c981e692519db605f0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329] to archive 2024-12-11T06:15:47,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/8f22dc3eb6c74086a6337fb78285664b 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/42f5c904a0a64392afc96d66b07ebf9d 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/535dfe2f8c5b494f8ab540f256507739 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/6a8f1d1bd0774f47ad835e5ae3602bf4 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2ba98f02ae34429dbecbd2465499ff63 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2ba98f02ae34429dbecbd2465499ff63 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/957d29a66ec34baabcbcd665a7ebf0d7 2024-12-11T06:15:47,596 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/b0c404f0a7f74e45a5081e2c004bfea4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/b0c404f0a7f74e45a5081e2c004bfea4 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/02fdde8a247342fc9eb6654c69fe1eda 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fcb8f60e032b4ae09902dea782b5ac09 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/09112a70726647a99e7b317360d593cb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/09112a70726647a99e7b317360d593cb 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/f9a634e32837464aa02e980889b720cf to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/f9a634e32837464aa02e980889b720cf 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4305ca7be70444e1b3a3bb1eb2c579c9 2024-12-11T06:15:47,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/ac48f8b3c3834ca3823177f9f383becf 2024-12-11T06:15:47,608 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/bc7f5c9457724213a6d6bd3abac045b0 2024-12-11T06:15:47,612 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2f27978e35a044b89bfc5e6e81b0f77e 2024-12-11T06:15:47,625 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/57a31a35a84c404cbef5aef55b90e8b0 2024-12-11T06:15:47,625 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/9cc6ce8481804732813ad88d1785ee58 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/9cc6ce8481804732813ad88d1785ee58 2024-12-11T06:15:47,626 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/706c0e9a09994da083ea5d0919a9da8a 2024-12-11T06:15:47,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fdc47ded4897485f9a993495920a1ab7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fdc47ded4897485f9a993495920a1ab7 2024-12-11T06:15:47,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/162d3b8899b04235a79b40b04ab8a487 2024-12-11T06:15:47,629 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/78a85d561865490490c451d3be30937f 2024-12-11T06:15:47,630 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/4fd8ea59ce214db498ac810bac3c407e 2024-12-11T06:15:47,631 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2b3357ffc1a64bdfbcab3f1703d986eb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/2b3357ffc1a64bdfbcab3f1703d986eb 2024-12-11T06:15:47,633 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/80f726bd192d4d5db691c49acc08ea93 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/80f726bd192d4d5db691c49acc08ea93 2024-12-11T06:15:47,633 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/3e11530773214a6d8bf351fef295f4c1 2024-12-11T06:15:47,634 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/253c07650b144e31986b7a917eb5cacb 2024-12-11T06:15:47,634 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/fd77acd07c3d4df68c8a17842ce7c1ae 2024-12-11T06:15:47,635 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/7b689b8e54ea4263bb6678c64063ee4d 2024-12-11T06:15:47,635 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/0d12d34f60fb48c981e692519db605f0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/0d12d34f60fb48c981e692519db605f0 2024-12-11T06:15:47,635 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1716ec5ace0441b993428ec8e0139a09 2024-12-11T06:15:47,636 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/60ffbfbfde0d412ca50ed28d3315f329 2024-12-11T06:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:47,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/1fde8462142d49c4a669587bbe59161b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/affa960e214740dbb6784512c40c5be9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b739b104c85d42f6b2819156efb8a888, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/7cdcce4a47fa4c1e81774e87880bbd3c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/5d3b1accbd0646c9b1ecaa186d121769, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2b1408ad0fb54e54b5e4e5546bfe17c6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ce1c1750b7bf462798e580b3e998006b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64dd2d3742084f84ab7c040932813390, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/560a28f04ca64095a3454887d93dcc12, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a] to archive 2024-12-11T06:15:47,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:15:47,652 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b34878a623534a34ac4c6336478ed896 2024-12-11T06:15:47,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f56ab391826f49ed8dee7229d46b2a7e 2024-12-11T06:15:47,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/1fde8462142d49c4a669587bbe59161b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/1fde8462142d49c4a669587bbe59161b 2024-12-11T06:15:47,657 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b739b104c85d42f6b2819156efb8a888 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b739b104c85d42f6b2819156efb8a888 2024-12-11T06:15:47,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9d4108006637465f92606a48c90f9eb2 2024-12-11T06:15:47,659 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/affa960e214740dbb6784512c40c5be9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/affa960e214740dbb6784512c40c5be9 2024-12-11T06:15:47,659 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/27f76ee04597464d9c700ea3f02af394 2024-12-11T06:15:47,661 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/9cc750aeb27d44eb851d93f5e1d33a2f 2024-12-11T06:15:47,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/09774403dfa14645bd29eb97ad8b4f8d 2024-12-11T06:15:47,661 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64ae19d0d3e840709ac43fe40d70cd0f 2024-12-11T06:15:47,661 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d6b8e6a43a674aeba3c3d0beaf1c616e 2024-12-11T06:15:47,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/41af954225724470aa26d5d8e3eb924b 2024-12-11T06:15:47,663 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ea6f76bb7e304d0d926a898fe980afce 2024-12-11T06:15:47,663 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/7cdcce4a47fa4c1e81774e87880bbd3c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/7cdcce4a47fa4c1e81774e87880bbd3c 2024-12-11T06:15:47,664 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/5d3b1accbd0646c9b1ecaa186d121769 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/5d3b1accbd0646c9b1ecaa186d121769 2024-12-11T06:15:47,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/eb40a5f043a84411b83aeb95836b37d4 2024-12-11T06:15:47,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2447db1c0df44c8b9d8184d7754f0d27 2024-12-11T06:15:47,665 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/4adba295cb4949aaab35631a24c04fcd 2024-12-11T06:15:47,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2b1408ad0fb54e54b5e4e5546bfe17c6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/2b1408ad0fb54e54b5e4e5546bfe17c6 2024-12-11T06:15:47,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ce1c1750b7bf462798e580b3e998006b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/ce1c1750b7bf462798e580b3e998006b 2024-12-11T06:15:47,666 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/c76e90ec07c14af9bf601ff55e0bcdf6 2024-12-11T06:15:47,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64dd2d3742084f84ab7c040932813390 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/64dd2d3742084f84ab7c040932813390 2024-12-11T06:15:47,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/8ad8830b847b46de88610f46b915cdb1 2024-12-11T06:15:47,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d21092baf9cb49e39c0934003012b1a5 2024-12-11T06:15:47,667 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/33b4bbd107af4bb5b270d40fc4ada0b4 2024-12-11T06:15:47,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/28e09095661049f69b61b20d1d2efd46 2024-12-11T06:15:47,669 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/b9c617cb7d6949c3bbb813cb929c781d 2024-12-11T06:15:47,669 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/560a28f04ca64095a3454887d93dcc12 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/560a28f04ca64095a3454887d93dcc12 2024-12-11T06:15:47,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/6c79a3b5e199483c964834122ec2ac1a 2024-12-11T06:15:47,669 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/d581ac006a7e49b0bb9dbdc6913ef1d1 2024-12-11T06:15:47,673 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3d3f0fbbd5f64699ab6131d68e3db11c 2024-12-11T06:15:47,693 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/recovered.edits/1050.seqid, newMaxSeqId=1050, maxSeqId=1 2024-12-11T06:15:47,697 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. 2024-12-11T06:15:47,697 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 682402d56c3b62fdcea61b4be01f333c: Waiting for close lock at 1733897746673Running coprocessor pre-close hooks at 1733897746673Disabling compacts and flushes for region at 1733897746673Disabling writes for close at 1733897746673Obtaining lock to block concurrent updates at 1733897746674 (+1 ms)Preparing flush snapshotting stores in 682402d56c3b62fdcea61b4be01f333c at 1733897746674Finished memstore snapshotting TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733897746674Flushing stores of TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c. at 1733897746674Flushing 682402d56c3b62fdcea61b4be01f333c/A: creating writer at 1733897746674Flushing 682402d56c3b62fdcea61b4be01f333c/A: appending metadata at 1733897746677 (+3 ms)Flushing 682402d56c3b62fdcea61b4be01f333c/A: closing flushed file at 1733897746677Flushing 682402d56c3b62fdcea61b4be01f333c/B: creating writer at 1733897747086 (+409 ms)Flushing 682402d56c3b62fdcea61b4be01f333c/B: appending metadata at 1733897747089 (+3 ms)Flushing 682402d56c3b62fdcea61b4be01f333c/B: closing flushed file at 1733897747089Flushing 682402d56c3b62fdcea61b4be01f333c/C: creating writer at 1733897747098 (+9 ms)Flushing 682402d56c3b62fdcea61b4be01f333c/C: appending metadata at 1733897747100 (+2 ms)Flushing 682402d56c3b62fdcea61b4be01f333c/C: closing flushed file at 1733897747100Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44904340: reopening flushed file at 1733897747519 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c87c517: reopening flushed file at 1733897747524 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f4d297: reopening flushed file at 1733897747530 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 682402d56c3b62fdcea61b4be01f333c in 864ms, sequenceid=1047, compaction requested=true at 1733897747537 (+7 ms)Writing region close event to WAL at 1733897747675 (+138 ms)Running coprocessor post-close hooks at 1733897747697 (+22 ms)Closed at 1733897747697 2024-12-11T06:15:47,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=682402d56c3b62fdcea61b4be01f333c, regionState=CLOSED 2024-12-11T06:15:47,711 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:47,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:47,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-11T06:15:47,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 682402d56c3b62fdcea61b4be01f333c, server=4518703b2c1d,35425,1733897658824 in 1.1970 sec 2024-12-11T06:15:47,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-11T06:15:47,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=682402d56c3b62fdcea61b4be01f333c, UNASSIGN in 1.2100 sec 2024-12-11T06:15:47,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-11T06:15:47,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2170 sec 2024-12-11T06:15:47,733 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897747733"}]},"ts":"1733897747733"} 2024-12-11T06:15:47,736 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:15:47,736 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:15:47,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-12-11T06:15:48,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-11T06:15:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T06:15:48,659 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:48,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:15:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,660 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T06:15:48,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:48,665 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/recovered.edits] 2024-12-11T06:15:48,668 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7658a24e84274e8c9d3b16931a20d0c3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/7658a24e84274e8c9d3b16931a20d0c3 2024-12-11T06:15:48,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c17ef9205ea54d39ab34daf6bf8e435e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/c17ef9205ea54d39ab34daf6bf8e435e 2024-12-11T06:15:48,668 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e6ff1ee4e8674bddb4a4e08288462718 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/A/e6ff1ee4e8674bddb4a4e08288462718 2024-12-11T06:15:48,671 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1d09cecf00a24823ab03830153ca4515 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/1d09cecf00a24823ab03830153ca4515 2024-12-11T06:15:48,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/d1c81da90234456a9d16686004df0d19 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/d1c81da90234456a9d16686004df0d19 2024-12-11T06:15:48,671 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/cc0dcb783e0c4d8f9d6daefe56bee16b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/B/cc0dcb783e0c4d8f9d6daefe56bee16b 2024-12-11T06:15:48,673 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/62a92fcf5fbc48028c5319553fe5479f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/62a92fcf5fbc48028c5319553fe5479f 2024-12-11T06:15:48,674 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3e1594cdadaf4c4cb81bd4ec9652658e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/3e1594cdadaf4c4cb81bd4ec9652658e 2024-12-11T06:15:48,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f24a1805a6534b53893394dd5e7e96b5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/C/f24a1805a6534b53893394dd5e7e96b5 2024-12-11T06:15:48,678 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/recovered.edits/1050.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c/recovered.edits/1050.seqid 2024-12-11T06:15:48,678 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/682402d56c3b62fdcea61b4be01f333c 2024-12-11T06:15:48,678 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:15:48,681 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,684 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:15:48,686 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:15:48,687 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,688 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:15:48,688 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897748688"}]},"ts":"9223372036854775807"} 2024-12-11T06:15:48,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:15:48,690 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 682402d56c3b62fdcea61b4be01f333c, NAME => 'TestAcidGuarantees,,1733897723949.682402d56c3b62fdcea61b4be01f333c.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:15:48,690 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:15:48,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897748690"}]},"ts":"9223372036854775807"} 2024-12-11T06:15:48,693 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:15:48,693 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-12-11T06:15:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T06:15:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:15:48,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:48,800 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=257 (was 257), OpenFileDescriptor=463 (was 461) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=849 (was 792) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=795 (was 277) - AvailableMemoryMB LEAK? - 2024-12-11T06:15:48,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=257, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=849, ProcessCount=11, AvailableMemoryMB=793 2024-12-11T06:15:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:15:48,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:15:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:48,817 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:15:48,817 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:48,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-11T06:15:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T06:15:48,819 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:15:48,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742145_1321 (size=1061) 2024-12-11T06:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T06:15:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T06:15:49,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:15:49,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742146_1322 (size=53) 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4087fec5b081d47691e33f5c7c3537f9, disabling compactions & flushes 2024-12-11T06:15:49,260 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. after waiting 0 ms 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,260 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4087fec5b081d47691e33f5c7c3537f9: Waiting for close lock at 1733897749260Disabling compacts and flushes for region at 1733897749260Disabling writes for close at 1733897749260Writing region close event to WAL at 1733897749260Closed at 1733897749260 2024-12-11T06:15:49,261 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:15:49,262 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897749261"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897749261"}]},"ts":"1733897749261"} 2024-12-11T06:15:49,263 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:15:49,264 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:15:49,264 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897749264"}]},"ts":"1733897749264"} 2024-12-11T06:15:49,265 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:15:49,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, ASSIGN}] 2024-12-11T06:15:49,267 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, ASSIGN 2024-12-11T06:15:49,267 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:15:49,418 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:49,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, ASSIGN because future has completed 2024-12-11T06:15:49,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T06:15:49,574 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,574 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:15:49,575 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,575 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:15:49,575 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,575 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,576 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,577 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:49,578 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName A 2024-12-11T06:15:49,578 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:49,578 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:49,578 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,579 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:49,579 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName B 2024-12-11T06:15:49,579 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:49,580 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:49,580 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,580 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:49,581 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName C 2024-12-11T06:15:49,581 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:49,581 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:49,581 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,581 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,582 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,582 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,583 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,583 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,584 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:15:49,585 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,586 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:15:49,587 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 4087fec5b081d47691e33f5c7c3537f9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67489545, jitterRate=0.00567258894443512}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:15:49,587 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:49,588 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 4087fec5b081d47691e33f5c7c3537f9: Running coprocessor pre-open hook at 1733897749575Writing region info on filesystem at 1733897749575Initializing all the Stores at 1733897749576 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897749576Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897749576Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897749576Cleaning up temporary data from old regions at 1733897749583 (+7 ms)Running coprocessor post-open hooks at 1733897749587 (+4 ms)Region opened successfully at 1733897749588 (+1 ms) 2024-12-11T06:15:49,589 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., pid=85, masterSystemTime=1733897749572 2024-12-11T06:15:49,591 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,591 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:49,591 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:49,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:49,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-11T06:15:49,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 in 173 msec 2024-12-11T06:15:49,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-11T06:15:49,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, ASSIGN in 328 msec 2024-12-11T06:15:49,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:15:49,596 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897749596"}]},"ts":"1733897749596"} 2024-12-11T06:15:49,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:15:49,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:15:49,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 783 msec 2024-12-11T06:15:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-11T06:15:49,959 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:49,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bdeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:49,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:49,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:49,962 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe732d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:49,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f79616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:49,965 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:49,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:49,968 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:15:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:15:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45b58a33 2024-12-11T06:15:49,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:15:49,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:15:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:15:49,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:15:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T06:15:49,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742147_1323 (size=1097) 2024-12-11T06:15:50,393 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T06:15:50,393 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T06:15:50,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:15:50,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE}] 2024-12-11T06:15:50,401 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE 2024-12-11T06:15:50,402 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:50,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE because future has completed 2024-12-11T06:15:50,405 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:15:50,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:50,556 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,556 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:15:50,556 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 4087fec5b081d47691e33f5c7c3537f9, disabling compactions & flushes 2024-12-11T06:15:50,557 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,557 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,557 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. after waiting 0 ms 2024-12-11T06:15:50,557 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,559 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T06:15:50,560 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,560 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 4087fec5b081d47691e33f5c7c3537f9: Waiting for close lock at 1733897750556Running coprocessor pre-close hooks at 1733897750556Disabling compacts and flushes for region at 1733897750556Disabling writes for close at 1733897750557 (+1 ms)Writing region close event to WAL at 1733897750557Running coprocessor post-close hooks at 1733897750560 (+3 ms)Closed at 1733897750560 2024-12-11T06:15:50,560 WARN [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 4087fec5b081d47691e33f5c7c3537f9 to self. 2024-12-11T06:15:50,562 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=CLOSED 2024-12-11T06:15:50,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:50,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-11T06:15:50,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 in 159 msec 2024-12-11T06:15:50,567 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE; state=CLOSED, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=true 2024-12-11T06:15:50,717 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:50,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE because future has completed 2024-12-11T06:15:50,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:15:50,873 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,873 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:15:50,873 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,874 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:15:50,874 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,874 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,875 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,875 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:50,875 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName A 2024-12-11T06:15:50,876 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:50,877 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:50,877 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,878 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:50,878 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName B 2024-12-11T06:15:50,878 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:50,878 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:50,878 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,879 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:15:50,879 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4087fec5b081d47691e33f5c7c3537f9 columnFamilyName C 2024-12-11T06:15:50,879 DEBUG [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:50,879 INFO [StoreOpener-4087fec5b081d47691e33f5c7c3537f9-1 {}] regionserver.HStore(327): Store=4087fec5b081d47691e33f5c7c3537f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:15:50,880 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,880 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,880 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,881 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,881 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,882 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,882 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:15:50,883 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,883 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 4087fec5b081d47691e33f5c7c3537f9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63211823, jitterRate=-0.058070436120033264}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:15:50,883 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,884 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 4087fec5b081d47691e33f5c7c3537f9: Running coprocessor pre-open hook at 1733897750874Writing region info on filesystem at 1733897750874Initializing all the Stores at 1733897750874Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897750874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897750874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897750874Cleaning up temporary data from old regions at 1733897750882 (+8 ms)Running coprocessor post-open hooks at 1733897750883 (+1 ms)Region opened successfully at 1733897750884 (+1 ms) 2024-12-11T06:15:50,884 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., pid=90, masterSystemTime=1733897750870 2024-12-11T06:15:50,886 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,886 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:50,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=OPEN, openSeqNum=5, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:50,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:15:50,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-11T06:15:50,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 in 169 msec 2024-12-11T06:15:50,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-11T06:15:50,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, REOPEN/MOVE in 489 msec 2024-12-11T06:15:50,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-11T06:15:50,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-12-11T06:15:50,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-12-11T06:15:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-11T06:15:50,895 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:50,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a2faf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a7254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b9090c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7123243b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b09591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f96682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3d97f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e242698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3948ed3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b117568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,928 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56591966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,929 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25fcdfeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,931 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac95074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,933 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef994b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,935 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b73b0bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d56d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf50695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e1d45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:15:50,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:15:50,941 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:15:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:15:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:15:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:15:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:15:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:15:50,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:15:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25544198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:15:50,945 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,945 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,947 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,948 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,948 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,958 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,959 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,960 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,960 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:50,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,964 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:15:50,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-11T06:15:50,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:15:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:15:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:15:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:15:50,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:15:50,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:50,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:50,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:50,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:50,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:50,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:50,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:50,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:51,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f1490b658a1548e59e82c39fc9d4c2d5_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897750970/Put/seqid=0 2024-12-11T06:15:51,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897811022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897811028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897811029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897811040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897811054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742148_1324 (size=12154) 2024-12-11T06:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:51,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897811149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897811149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897811158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897811169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897811178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:51,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897811373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897811374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897811378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897811399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897811399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:51,490 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f1490b658a1548e59e82c39fc9d4c2d5_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f1490b658a1548e59e82c39fc9d4c2d5_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:51,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/aa697565116f4b1c99605f2411998ba6, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:51,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/aa697565116f4b1c99605f2411998ba6 is 175, key is test_row_0/A:col10/1733897750970/Put/seqid=0 2024-12-11T06:15:51,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742149_1325 (size=30955) 2024-12-11T06:15:51,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897811693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897811693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897811699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897811710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897811710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:51,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:51,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:51,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:51,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:51,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/aa697565116f4b1c99605f2411998ba6 2024-12-11T06:15:52,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0d356847bc854829a2a50be55828ee81 is 50, key is test_row_0/B:col10/1733897750970/Put/seqid=0 2024-12-11T06:15:52,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742150_1326 (size=12001) 2024-12-11T06:15:52,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:52,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897812209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897812221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:52,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897812221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897812228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:52,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897812228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:52,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0d356847bc854829a2a50be55828ee81 2024-12-11T06:15:52,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/da7698d6ae994abc905c159534132874 is 50, key is test_row_0/C:col10/1733897750970/Put/seqid=0 2024-12-11T06:15:52,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742151_1327 (size=12001) 2024-12-11T06:15:52,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:52,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:52,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/da7698d6ae994abc905c159534132874 2024-12-11T06:15:52,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/aa697565116f4b1c99605f2411998ba6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6 2024-12-11T06:15:52,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6, entries=150, sequenceid=16, filesize=30.2 K 2024-12-11T06:15:52,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0d356847bc854829a2a50be55828ee81 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81 2024-12-11T06:15:52,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81, entries=150, sequenceid=16, filesize=11.7 K 2024-12-11T06:15:53,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/da7698d6ae994abc905c159534132874 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874 2024-12-11T06:15:53,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874, entries=150, sequenceid=16, filesize=11.7 K 2024-12-11T06:15:53,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4087fec5b081d47691e33f5c7c3537f9 in 2038ms, sequenceid=16, compaction requested=false 2024-12-11T06:15:53,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:53,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-11T06:15:53,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:53,045 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412112b7416753b7a4b66b5aa8d94f220cc23_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897751036/Put/seqid=0 2024-12-11T06:15:53,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742152_1328 (size=12154) 2024-12-11T06:15:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:53,113 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412112b7416753b7a4b66b5aa8d94f220cc23_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112b7416753b7a4b66b5aa8d94f220cc23_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/ab6984b2fecc4c0d8bfc3d3da6a392ad, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/ab6984b2fecc4c0d8bfc3d3da6a392ad is 175, key is test_row_0/A:col10/1733897751036/Put/seqid=0 2024-12-11T06:15:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:53,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742153_1329 (size=30955) 2024-12-11T06:15:53,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/ab6984b2fecc4c0d8bfc3d3da6a392ad 2024-12-11T06:15:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/20acd5ca02f74e35a56517307600678d is 50, key is test_row_0/B:col10/1733897751036/Put/seqid=0 2024-12-11T06:15:53,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742154_1330 (size=12001) 2024-12-11T06:15:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/20acd5ca02f74e35a56517307600678d 2024-12-11T06:15:53,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d6d45800bc14325878d1dd91475a69a is 50, key is test_row_0/C:col10/1733897751036/Put/seqid=0 2024-12-11T06:15:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742155_1331 (size=12001) 2024-12-11T06:15:53,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897813252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897813254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897813260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897813262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897813266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,288 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:15:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897813379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897813379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897813380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897813380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897813388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897813600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897813601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897813602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897813605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897813609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,704 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d6d45800bc14325878d1dd91475a69a 2024-12-11T06:15:53,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/ab6984b2fecc4c0d8bfc3d3da6a392ad as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad 2024-12-11T06:15:53,728 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad, entries=150, sequenceid=36, filesize=30.2 K 2024-12-11T06:15:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/20acd5ca02f74e35a56517307600678d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d 2024-12-11T06:15:53,736 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d, entries=150, sequenceid=36, filesize=11.7 K 2024-12-11T06:15:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d6d45800bc14325878d1dd91475a69a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a 2024-12-11T06:15:53,748 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a, entries=150, sequenceid=36, filesize=11.7 K 2024-12-11T06:15:53,753 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4087fec5b081d47691e33f5c7c3537f9 in 708ms, sequenceid=36, compaction requested=false 2024-12-11T06:15:53,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-11T06:15:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-11T06:15:53,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-11T06:15:53,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7820 sec 2024-12-11T06:15:53,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.7980 sec 2024-12-11T06:15:53,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:53,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113e92223c9e704f8eba2eb09fee5029f4_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:53,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742156_1332 (size=14594) 2024-12-11T06:15:53,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897813977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897813978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897813980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897813987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897813985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897814099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897814099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897814100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897814110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897814110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897814319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897814319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897814321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897814321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897814328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:54,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113e92223c9e704f8eba2eb09fee5029f4_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113e92223c9e704f8eba2eb09fee5029f4_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:54,371 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/51a8d947f0e442fe982bdb718ed8fe8d, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:54,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/51a8d947f0e442fe982bdb718ed8fe8d is 175, key is test_row_0/A:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:54,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742157_1333 (size=39549) 2024-12-11T06:15:54,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/51a8d947f0e442fe982bdb718ed8fe8d 2024-12-11T06:15:54,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/e5dc55c7252346ab9e48e1616b94a317 is 50, key is test_row_0/B:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:54,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742158_1334 (size=12001) 2024-12-11T06:15:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897814629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897814641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897814645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897814645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897814659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:54,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:54,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/e5dc55c7252346ab9e48e1616b94a317 2024-12-11T06:15:54,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/21ae86bb050f4b7493b4b450a0f9b65c is 50, key is test_row_0/C:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:54,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742159_1335 (size=12001) 2024-12-11T06:15:54,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/21ae86bb050f4b7493b4b450a0f9b65c 2024-12-11T06:15:54,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/51a8d947f0e442fe982bdb718ed8fe8d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d 2024-12-11T06:15:55,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d, entries=200, sequenceid=49, filesize=38.6 K 2024-12-11T06:15:55,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/e5dc55c7252346ab9e48e1616b94a317 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317 2024-12-11T06:15:55,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317, entries=150, sequenceid=49, filesize=11.7 K 2024-12-11T06:15:55,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/21ae86bb050f4b7493b4b450a0f9b65c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c 2024-12-11T06:15:55,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c, entries=150, sequenceid=49, filesize=11.7 K 2024-12-11T06:15:55,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4087fec5b081d47691e33f5c7c3537f9 in 1113ms, sequenceid=49, compaction requested=true 2024-12-11T06:15:55,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:55,042 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:55,042 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:55,043 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:55,044 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:15:55,044 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,044 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=35.2 K 2024-12-11T06:15:55,044 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:55,044 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:15:55,044 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,044 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=99.1 K 2024-12-11T06:15:55,044 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,044 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d] 2024-12-11T06:15:55,045 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d356847bc854829a2a50be55828ee81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733897750963 2024-12-11T06:15:55,045 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa697565116f4b1c99605f2411998ba6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733897750963 2024-12-11T06:15:55,046 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6984b2fecc4c0d8bfc3d3da6a392ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733897751006 2024-12-11T06:15:55,046 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 20acd5ca02f74e35a56517307600678d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733897751006 2024-12-11T06:15:55,046 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a8d947f0e442fe982bdb718ed8fe8d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:55,047 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dc55c7252346ab9e48e1616b94a317, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:55,065 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:55,086 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:55,086 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/e7026008f02e4e6a9c3fa3fe669f75f9 is 50, key is test_row_0/B:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:55,092 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412111f6deb37c296475f804a8b6e7e82adf8_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:55,094 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412111f6deb37c296475f804a8b6e7e82adf8_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:55,095 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111f6deb37c296475f804a8b6e7e82adf8_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-11T06:15:55,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:55,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-11T06:15:55,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:55,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:55,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:55,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742160_1336 (size=12104) 2024-12-11T06:15:55,169 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/e7026008f02e4e6a9c3fa3fe669f75f9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e7026008f02e4e6a9c3fa3fe669f75f9 2024-12-11T06:15:55,175 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into e7026008f02e4e6a9c3fa3fe669f75f9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:55,175 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:55,175 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897755042; duration=0sec 2024-12-11T06:15:55,176 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:55,176 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:15:55,176 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:15:55,177 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:15:55,177 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:15:55,177 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,177 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=35.2 K 2024-12-11T06:15:55,178 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting da7698d6ae994abc905c159534132874, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733897750963 2024-12-11T06:15:55,179 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6d45800bc14325878d1dd91475a69a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733897751006 2024-12-11T06:15:55,179 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ae86bb050f4b7493b4b450a0f9b65c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:55,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742161_1337 (size=4469) 2024-12-11T06:15:55,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d0f4f05992f74ae9bd9b7816e64a03f8_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897753978/Put/seqid=0 2024-12-11T06:15:55,215 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#288 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:55,216 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b9e6abd78f2f4f838335482215ec4492 is 175, key is test_row_0/A:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897815208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897815213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897815213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897815222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897815239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,257 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#291 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:55,258 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/d51a5cd2343c426dbfef54545bbb7589 is 50, key is test_row_0/C:col10/1733897753247/Put/seqid=0 2024-12-11T06:15:55,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742162_1338 (size=31058) 2024-12-11T06:15:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742163_1339 (size=14594) 2024-12-11T06:15:55,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742164_1340 (size=12104) 2024-12-11T06:15:55,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:55,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,312 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/d51a5cd2343c426dbfef54545bbb7589 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/d51a5cd2343c426dbfef54545bbb7589 2024-12-11T06:15:55,328 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into d51a5cd2343c426dbfef54545bbb7589(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:55,328 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:55,329 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897755042; duration=0sec 2024-12-11T06:15:55,329 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:55,329 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:15:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897815351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897815351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897815351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897815351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897815358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897815569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897815569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897815569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897815584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897815584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,684 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b9e6abd78f2f4f838335482215ec4492 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492 2024-12-11T06:15:55,688 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:55,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d0f4f05992f74ae9bd9b7816e64a03f8_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0f4f05992f74ae9bd9b7816e64a03f8_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:55,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/96fc17b5914841748cb3451ba27a021e, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:55,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/96fc17b5914841748cb3451ba27a021e is 175, key is test_row_0/A:col10/1733897753978/Put/seqid=0 2024-12-11T06:15:55,697 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into b9e6abd78f2f4f838335482215ec4492(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:55,697 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:55,697 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897755042; duration=0sec 2024-12-11T06:15:55,697 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:55,697 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:15:55,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742165_1341 (size=39549) 2024-12-11T06:15:55,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:55,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:55,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897815889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897815889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897815889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897815901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:55,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:55,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897815910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:55,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/96fc17b5914841748cb3451ba27a021e 2024-12-11T06:15:56,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/03590cdf02774938b16c9103efd86bc2 is 50, key is test_row_0/B:col10/1733897753978/Put/seqid=0 2024-12-11T06:15:56,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:56,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742166_1342 (size=12001) 2024-12-11T06:15:56,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/03590cdf02774938b16c9103efd86bc2 2024-12-11T06:15:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:56,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d723c787c4b452fb1946c12b6e7b33c is 50, key is test_row_0/C:col10/1733897753978/Put/seqid=0 2024-12-11T06:15:56,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742167_1343 (size=12001) 2024-12-11T06:15:56,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897816408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897816409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897816419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897816419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897816428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:56,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:56,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:56,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d723c787c4b452fb1946c12b6e7b33c 2024-12-11T06:15:56,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/96fc17b5914841748cb3451ba27a021e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e 2024-12-11T06:15:56,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e, entries=200, sequenceid=70, filesize=38.6 K 2024-12-11T06:15:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/03590cdf02774938b16c9103efd86bc2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2 2024-12-11T06:15:56,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2, entries=150, sequenceid=70, filesize=11.7 K 2024-12-11T06:15:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/9d723c787c4b452fb1946c12b6e7b33c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c 2024-12-11T06:15:56,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c, entries=150, sequenceid=70, filesize=11.7 K 2024-12-11T06:15:56,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4087fec5b081d47691e33f5c7c3537f9 in 1581ms, sequenceid=70, compaction requested=false 2024-12-11T06:15:56,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:56,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:56,856 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:56,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115b6e7e485fe540e2948db9003fe908be_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897755181/Put/seqid=0 2024-12-11T06:15:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742168_1344 (size=12154) 2024-12-11T06:15:56,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:56,952 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115b6e7e485fe540e2948db9003fe908be_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b6e7e485fe540e2948db9003fe908be_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:56,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/9fbf9e08067c4e60b32f245535c4078d, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:56,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/9fbf9e08067c4e60b32f245535c4078d is 175, key is test_row_0/A:col10/1733897755181/Put/seqid=0 2024-12-11T06:15:57,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742169_1345 (size=30955) 2024-12-11T06:15:57,010 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/9fbf9e08067c4e60b32f245535c4078d 2024-12-11T06:15:57,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/9d22160c7bc2430e89dc9a3eafa5629b is 50, key is test_row_0/B:col10/1733897755181/Put/seqid=0 2024-12-11T06:15:57,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742170_1346 (size=12001) 2024-12-11T06:15:57,084 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/9d22160c7bc2430e89dc9a3eafa5629b 2024-12-11T06:15:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/292a58969f8a4020b5ac2b03f70e40bc is 50, key is test_row_0/C:col10/1733897755181/Put/seqid=0 2024-12-11T06:15:57,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742171_1347 (size=12001) 2024-12-11T06:15:57,157 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/292a58969f8a4020b5ac2b03f70e40bc 2024-12-11T06:15:57,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/9fbf9e08067c4e60b32f245535c4078d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d 2024-12-11T06:15:57,201 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d, entries=150, sequenceid=84, filesize=30.2 K 2024-12-11T06:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/9d22160c7bc2430e89dc9a3eafa5629b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b 2024-12-11T06:15:57,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b, entries=150, sequenceid=84, filesize=11.7 K 2024-12-11T06:15:57,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/292a58969f8a4020b5ac2b03f70e40bc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc 2024-12-11T06:15:57,225 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc, entries=150, sequenceid=84, filesize=11.7 K 2024-12-11T06:15:57,226 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 4087fec5b081d47691e33f5c7c3537f9 in 370ms, sequenceid=84, compaction requested=true 2024-12-11T06:15:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-11T06:15:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-11T06:15:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-11T06:15:57,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-12-11T06:15:57,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1030 sec 2024-12-11T06:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-11T06:15:57,298 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:57,299 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-11T06:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,301 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,302 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T06:15:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T06:15:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-11T06:15:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-11T06:15:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-11T06:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-11T06:15:57,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 161 msec 2024-12-11T06:15:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 168 msec 2024-12-11T06:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-11T06:15:57,619 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:15:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,619 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-11T06:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:15:57,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:57,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:15:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bee52102741844d2819014bf2e555dc2_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:57,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742172_1348 (size=17034) 2024-12-11T06:15:57,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:57,743 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bee52102741844d2819014bf2e555dc2_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bee52102741844d2819014bf2e555dc2_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:57,744 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f46b88458308417b845fa10abd1f3e89, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:57,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f46b88458308417b845fa10abd1f3e89 is 175, key is test_row_0/A:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:57,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742173_1349 (size=48135) 2024-12-11T06:15:57,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897817758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897817769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897817758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897757748, max timestamp=1733897757776Num uniques 859; 2024-12-11T06:15:57,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897757748, max timestamp=1733897757776Num uniques 859; 2024-12-11T06:15:57,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897757748, max timestamp=1733897757776Num uniques 859; 2024-12-11T06:15:57,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:57,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:15:57,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897817882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897817888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897817888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897817888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897817890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:57,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:57,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:57,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897817999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897818018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897818109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897818109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897818109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-11T06:15:58,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-11T06:15:58,162 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f46b88458308417b845fa10abd1f3e89 2024-12-11T06:15:58,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/027f4baa9f484c3a96d43d5ab740226a is 50, key is test_row_0/B:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:58,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742174_1350 (size=9657) 2024-12-11T06:15:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897818209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897818238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:58,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897818418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897818418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897818418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897818518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897818559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/027f4baa9f484c3a96d43d5ab740226a 2024-12-11T06:15:58,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c8d401790a0b4711a0aa57602b191c58 is 50, key is test_row_0/C:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:58,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742175_1351 (size=9657) 2024-12-11T06:15:58,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:58,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:58,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:58,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897818939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897818939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897818940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:59,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:59,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:15:59,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:59,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:15:59,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897819038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:59,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c8d401790a0b4711a0aa57602b191c58 2024-12-11T06:15:59,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:15:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897819083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:15:59,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f46b88458308417b845fa10abd1f3e89 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89 2024-12-11T06:15:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:15:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:15:59,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89, entries=250, sequenceid=96, filesize=47.0 K 2024-12-11T06:15:59,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/027f4baa9f484c3a96d43d5ab740226a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a 2024-12-11T06:15:59,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a, entries=100, sequenceid=96, filesize=9.4 K 2024-12-11T06:15:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c8d401790a0b4711a0aa57602b191c58 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58 2024-12-11T06:15:59,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58, entries=100, sequenceid=96, filesize=9.4 K 2024-12-11T06:15:59,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4087fec5b081d47691e33f5c7c3537f9 in 1435ms, sequenceid=96, compaction requested=true 2024-12-11T06:15:59,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:59,124 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:59,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:15:59,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:59,124 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:15:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:59,126 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149697 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:59,126 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:15:59,126 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,127 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=146.2 K 2024-12-11T06:15:59,127 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,127 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89] 2024-12-11T06:15:59,127 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e6abd78f2f4f838335482215ec4492, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:59,128 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:59,128 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:15:59,128 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,128 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e7026008f02e4e6a9c3fa3fe669f75f9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=44.7 K 2024-12-11T06:15:59,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fc17b5914841748cb3451ba27a021e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733897753966 2024-12-11T06:15:59,129 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e7026008f02e4e6a9c3fa3fe669f75f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:59,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fbf9e08067c4e60b32f245535c4078d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897755181 2024-12-11T06:15:59,129 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 03590cdf02774938b16c9103efd86bc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733897753966 2024-12-11T06:15:59,130 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f46b88458308417b845fa10abd1f3e89, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897757563 2024-12-11T06:15:59,130 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d22160c7bc2430e89dc9a3eafa5629b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897755181 2024-12-11T06:15:59,131 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 027f4baa9f484c3a96d43d5ab740226a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897757686 2024-12-11T06:15:59,144 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:59,156 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:59,158 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/ccbd2fa72f4a414a913a693685cf480f is 50, key is test_row_0/B:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:59,165 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211a9aaa34c734144848a7404829d937632_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:59,168 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211a9aaa34c734144848a7404829d937632_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:59,168 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a9aaa34c734144848a7404829d937632_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:59,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T06:15:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,188 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-11T06:15:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:15:59,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:59,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:15:59,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:59,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:15:59,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:15:59,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742177_1353 (size=4469) 2024-12-11T06:15:59,224 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#300 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:59,225 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/7a79663f8b9940a29a7b5b1e292f605f is 175, key is test_row_0/A:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:59,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742176_1352 (size=12241) 2024-12-11T06:15:59,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113885fb512a3f4944babba820f474bdec_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897757887/Put/seqid=0 2024-12-11T06:15:59,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742179_1355 (size=12304) 2024-12-11T06:15:59,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742178_1354 (size=31302) 2024-12-11T06:15:59,653 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/ccbd2fa72f4a414a913a693685cf480f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ccbd2fa72f4a414a913a693685cf480f 2024-12-11T06:15:59,671 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into ccbd2fa72f4a414a913a693685cf480f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:59,671 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:59,672 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=12, startTime=1733897759124; duration=0sec 2024-12-11T06:15:59,672 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:15:59,675 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:15:59,675 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:15:59,676 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:15:59,677 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:15:59,677 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:15:59,677 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/d51a5cd2343c426dbfef54545bbb7589, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=44.7 K 2024-12-11T06:15:59,683 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d51a5cd2343c426dbfef54545bbb7589, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733897753247 2024-12-11T06:15:59,684 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d723c787c4b452fb1946c12b6e7b33c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733897753966 2024-12-11T06:15:59,684 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 292a58969f8a4020b5ac2b03f70e40bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897755181 2024-12-11T06:15:59,684 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d401790a0b4711a0aa57602b191c58, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897757686 2024-12-11T06:15:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:15:59,701 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113885fb512a3f4944babba820f474bdec_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113885fb512a3f4944babba820f474bdec_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/624f16b9a39944cdb915dae88f3e9341, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:15:59,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/624f16b9a39944cdb915dae88f3e9341 is 175, key is test_row_0/A:col10/1733897757887/Put/seqid=0 2024-12-11T06:15:59,719 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/7a79663f8b9940a29a7b5b1e292f605f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f 2024-12-11T06:15:59,727 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into 7a79663f8b9940a29a7b5b1e292f605f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:59,727 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:59,727 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=12, startTime=1733897759124; duration=0sec 2024-12-11T06:15:59,727 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:59,727 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:15:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742180_1356 (size=31105) 2024-12-11T06:15:59,730 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:15:59,731 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/108a6e2ea4da4d23837a9c79d38ef28d is 50, key is test_row_0/C:col10/1733897757686/Put/seqid=0 2024-12-11T06:15:59,731 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/624f16b9a39944cdb915dae88f3e9341 2024-12-11T06:15:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/96dec828976448549d8c69ae229aa73e is 50, key is test_row_0/B:col10/1733897757887/Put/seqid=0 2024-12-11T06:15:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:15:59,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742181_1357 (size=12241) 2024-12-11T06:15:59,805 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/108a6e2ea4da4d23837a9c79d38ef28d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/108a6e2ea4da4d23837a9c79d38ef28d 2024-12-11T06:15:59,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742182_1358 (size=12151) 2024-12-11T06:15:59,814 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into 108a6e2ea4da4d23837a9c79d38ef28d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:15:59,814 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:15:59,814 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=12, startTime=1733897759125; duration=0sec 2024-12-11T06:15:59,814 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:15:59,814 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:15:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:15:59,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897820000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897820002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897820011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897820059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,067 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897820098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,108 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897820119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897820119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897820128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,217 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/96dec828976448549d8c69ae229aa73e 2024-12-11T06:16:00,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/813b0cf1c47a4fe992b5f6b41e8348f0 is 50, key is test_row_0/C:col10/1733897757887/Put/seqid=0 2024-12-11T06:16:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742183_1359 (size=12151) 2024-12-11T06:16:00,308 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/813b0cf1c47a4fe992b5f6b41e8348f0 2024-12-11T06:16:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/624f16b9a39944cdb915dae88f3e9341 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341 2024-12-11T06:16:00,358 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341, entries=150, sequenceid=132, filesize=30.4 K 2024-12-11T06:16:00,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/96dec828976448549d8c69ae229aa73e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e 2024-12-11T06:16:00,368 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e, entries=150, sequenceid=132, filesize=11.9 K 2024-12-11T06:16:00,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/813b0cf1c47a4fe992b5f6b41e8348f0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0 2024-12-11T06:16:00,377 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0, entries=150, sequenceid=132, filesize=11.9 K 2024-12-11T06:16:00,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4087fec5b081d47691e33f5c7c3537f9 in 1190ms, sequenceid=132, compaction requested=false 2024-12-11T06:16:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-11T06:16:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-11T06:16:00,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-11T06:16:00,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7580 sec 2024-12-11T06:16:00,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.7640 sec 2024-12-11T06:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:00,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:00,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:00,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121147627b885f844235943af50be4667814_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:00,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742184_1360 (size=14794) 2024-12-11T06:16:00,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:00,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121147627b885f844235943af50be4667814_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147627b885f844235943af50be4667814_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:00,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c62d9eeca4b4e1e88126de81a055dd2, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:00,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c62d9eeca4b4e1e88126de81a055dd2 is 175, key is test_row_0/A:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:00,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897820489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897820493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897820503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742185_1361 (size=39749) 2024-12-11T06:16:00,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897820609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897820619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897820619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897820829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897820829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897820841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:00,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c62d9eeca4b4e1e88126de81a055dd2 2024-12-11T06:16:00,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/ce24139179fb466e88057bd71bd8c549 is 50, key is test_row_0/B:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:01,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742186_1362 (size=12151) 2024-12-11T06:16:01,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/ce24139179fb466e88057bd71bd8c549 2024-12-11T06:16:01,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/189b0b5f7dd94d6f9ba4b93f287c11fe is 50, key is test_row_0/C:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742187_1363 (size=12151) 2024-12-11T06:16:01,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897821148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897821148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:01,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897821158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:01,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/189b0b5f7dd94d6f9ba4b93f287c11fe 2024-12-11T06:16:01,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c62d9eeca4b4e1e88126de81a055dd2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2 2024-12-11T06:16:01,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2, entries=200, sequenceid=147, filesize=38.8 K 2024-12-11T06:16:01,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/ce24139179fb466e88057bd71bd8c549 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549 2024-12-11T06:16:01,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549, entries=150, sequenceid=147, filesize=11.9 K 2024-12-11T06:16:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/189b0b5f7dd94d6f9ba4b93f287c11fe as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe 2024-12-11T06:16:01,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe, entries=150, sequenceid=147, filesize=11.9 K 2024-12-11T06:16:01,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4087fec5b081d47691e33f5c7c3537f9 in 1157ms, sequenceid=147, compaction requested=true 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:01,543 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:01,543 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:01,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:01,544 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:01,544 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:16:01,544 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,545 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=99.8 K 2024-12-11T06:16:01,545 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,545 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2] 2024-12-11T06:16:01,553 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:01,553 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:16:01,553 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,553 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ccbd2fa72f4a414a913a693685cf480f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=35.7 K 2024-12-11T06:16:01,553 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a79663f8b9940a29a7b5b1e292f605f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897755206 2024-12-11T06:16:01,554 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624f16b9a39944cdb915dae88f3e9341, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733897757873 2024-12-11T06:16:01,554 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ccbd2fa72f4a414a913a693685cf480f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897755206 2024-12-11T06:16:01,554 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c62d9eeca4b4e1e88126de81a055dd2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:01,554 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 96dec828976448549d8c69ae229aa73e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733897757873 2024-12-11T06:16:01,554 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ce24139179fb466e88057bd71bd8c549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:01,571 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:01,591 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:01,592 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/6553abe7abe64735b9eebf2b35eb4b42 is 50, key is test_row_0/B:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:01,601 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121149422b4f68e94263a0e1ababb6d82306_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:01,603 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121149422b4f68e94263a0e1ababb6d82306_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:01,603 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121149422b4f68e94263a0e1ababb6d82306_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:01,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742188_1364 (size=12493) 2024-12-11T06:16:01,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742189_1365 (size=4469) 2024-12-11T06:16:01,640 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#309 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:01,641 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/df01b9cf155b4256a9c4fe3ef3044c3b is 175, key is test_row_0/A:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:01,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742190_1366 (size=31447) 2024-12-11T06:16:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:01,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897760487, max timestamp=1733897761680Num uniques 859; 2024-12-11T06:16:01,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:01,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897760487, max timestamp=1733897761680Num uniques 859; 2024-12-11T06:16:01,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:01,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:01,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897760487, max timestamp=1733897761680Num uniques 859; 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:01,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:01,697 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/df01b9cf155b4256a9c4fe3ef3044c3b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b 2024-12-11T06:16:01,707 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into df01b9cf155b4256a9c4fe3ef3044c3b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:01,707 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:01,707 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897761543; duration=0sec 2024-12-11T06:16:01,707 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:01,707 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:16:01,707 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:01,708 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:01,708 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:16:01,708 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,708 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/108a6e2ea4da4d23837a9c79d38ef28d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=35.7 K 2024-12-11T06:16:01,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114a6321fabba54ac5ba43a4b85c673b22_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897761684/Put/seqid=0 2024-12-11T06:16:01,713 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 108a6e2ea4da4d23837a9c79d38ef28d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733897755206 2024-12-11T06:16:01,721 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813b0cf1c47a4fe992b5f6b41e8348f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733897757873 2024-12-11T06:16:01,725 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 189b0b5f7dd94d6f9ba4b93f287c11fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:01,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742191_1367 (size=19774) 2024-12-11T06:16:01,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:01,754 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114a6321fabba54ac5ba43a4b85c673b22_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114a6321fabba54ac5ba43a4b85c673b22_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:01,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bf01b4bf4a264efe9eefa0ccf6b83486, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:01,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bf01b4bf4a264efe9eefa0ccf6b83486 is 175, key is test_row_0/A:col10/1733897761684/Put/seqid=0 2024-12-11T06:16:01,763 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:01,764 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/a5ef90d7dd76463882696edc22fd5d61 is 50, key is test_row_0/C:col10/1733897760381/Put/seqid=0 2024-12-11T06:16:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T06:16:01,779 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:01,779 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-11T06:16:01,782 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:01,784 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:01,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:01,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897761728, max timestamp=1733897761792Num uniques 859; 2024-12-11T06:16:01,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:01,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897761728, max timestamp=1733897761792Num uniques 859; 2024-12-11T06:16:01,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897761728, max timestamp=1733897761792Num uniques 859; 2024-12-11T06:16:01,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:01,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742192_1368 (size=57033) 2024-12-11T06:16:01,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bf01b4bf4a264efe9eefa0ccf6b83486 2024-12-11T06:16:01,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742193_1369 (size=12493) 2024-12-11T06:16:01,846 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/a5ef90d7dd76463882696edc22fd5d61 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/a5ef90d7dd76463882696edc22fd5d61 2024-12-11T06:16:01,854 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into a5ef90d7dd76463882696edc22fd5d61(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:01,854 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:01,854 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897761543; duration=0sec 2024-12-11T06:16:01,855 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:01,855 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:16:01,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/55e8393f951541348e960f6a2971faa9 is 50, key is test_row_0/B:col10/1733897761684/Put/seqid=0 2024-12-11T06:16:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742194_1370 (size=12151) 2024-12-11T06:16:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:01,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897761783, max timestamp=1733897761910Num uniques 1068; 2024-12-11T06:16:01,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-11T06:16:01,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897761783, max timestamp=1733897761910Num uniques 1068; 2024-12-11T06:16:01,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-11T06:16:01,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897761783, max timestamp=1733897761910Num uniques 1068; 2024-12-11T06:16:01,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-11T06:16:01,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:01,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:01,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,041 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/6553abe7abe64735b9eebf2b35eb4b42 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/6553abe7abe64735b9eebf2b35eb4b42 2024-12-11T06:16:02,049 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into 6553abe7abe64735b9eebf2b35eb4b42(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:02,049 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:02,049 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897761543; duration=0sec 2024-12-11T06:16:02,049 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:02,049 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:16:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897822027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897762010, max timestamp=1733897762051Num uniques 1077; 2024-12-11T06:16:02,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897762010, max timestamp=1733897762051Num uniques 1077; 2024-12-11T06:16:02,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T06:16:02,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T06:16:02,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733897762010, max timestamp=1733897762051Num uniques 1077; 2024-12-11T06:16:02,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T06:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:02,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897822164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897762113, max timestamp=1733897762183Num uniques 1086; 2024-12-11T06:16:02,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-12-11T06:16:02,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897822166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897762113, max timestamp=1733897762184Num uniques 1086; 2024-12-11T06:16:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897762113, max timestamp=1733897762184Num uniques 1086; 2024-12-11T06:16:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-12-11T06:16:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-12-11T06:16:02,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/55e8393f951541348e960f6a2971faa9 2024-12-11T06:16:02,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/465f1c8d1e184926aeb8251a3e8e7ac7 is 50, key is test_row_0/C:col10/1733897761684/Put/seqid=0 2024-12-11T06:16:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742195_1371 (size=12151) 2024-12-11T06:16:02,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733897762212, max timestamp=1733897762327Num uniques 1009; 2024-12-11T06:16:02,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-12-11T06:16:02,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733897762212, max timestamp=1733897762327Num uniques 1009; 2024-12-11T06:16:02,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733897762212, max timestamp=1733897762327Num uniques 1009; 2024-12-11T06:16:02,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-12-11T06:16:02,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=8930, suffix heap size=132904,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=195 2024-12-11T06:16:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897822366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897822368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733897762352, max timestamp=1733897762381Num uniques 1054; 2024-12-11T06:16:02,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-11T06:16:02,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733897762352, max timestamp=1733897762381Num uniques 1054; 2024-12-11T06:16:02,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-11T06:16:02,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733897762352, max timestamp=1733897762381Num uniques 1054; 2024-12-11T06:16:02,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-11T06:16:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:02,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897822457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897822466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897822469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897822489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897822489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897822579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897822589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897822589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897822709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897822709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/465f1c8d1e184926aeb8251a3e8e7ac7 2024-12-11T06:16:02,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bf01b4bf4a264efe9eefa0ccf6b83486 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486 2024-12-11T06:16:02,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486, entries=300, sequenceid=170, filesize=55.7 K 2024-12-11T06:16:02,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/55e8393f951541348e960f6a2971faa9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9 2024-12-11T06:16:02,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9, entries=150, sequenceid=170, filesize=11.9 K 2024-12-11T06:16:02,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/465f1c8d1e184926aeb8251a3e8e7ac7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7 2024-12-11T06:16:02,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7, entries=150, sequenceid=170, filesize=11.9 K 2024-12-11T06:16:02,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 4087fec5b081d47691e33f5c7c3537f9 in 1067ms, sequenceid=170, compaction requested=false 2024-12-11T06:16:02,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:02,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=154.31 KB heapSize=436.55 KB 2024-12-11T06:16:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:02,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733897762454, max timestamp=1733897762807Num uniques 1113; 2024-12-11T06:16:02,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733897762454, max timestamp=1733897762807Num uniques 1113; 2024-12-11T06:16:02,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-11T06:16:02,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:16:02,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121128275348a4964478b5a3ec7c662dac35_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:02,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742196_1372 (size=14994) 2024-12-11T06:16:02,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:02,870 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121128275348a4964478b5a3ec7c662dac35_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128275348a4964478b5a3ec7c662dac35_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:02,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/e960dabb066c4a2ead3147dc055bc227, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:02,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/e960dabb066c4a2ead3147dc055bc227 is 175, key is test_row_0/A:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:02,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:02,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:02,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:02,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897822883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897822887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897822890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:02,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742197_1373 (size=39949) 2024-12-11T06:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:03,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897822999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897823009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897823009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897823029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897823029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897823219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897823219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897823219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,312 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/e960dabb066c4a2ead3147dc055bc227 2024-12-11T06:16:03,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/109c65b2d6c344249e27aa6da69c7cf9 is 50, key is test_row_0/B:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:03,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742198_1374 (size=12301) 2024-12-11T06:16:03,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/109c65b2d6c344249e27aa6da69c7cf9 2024-12-11T06:16:03,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/eba4145e716f4dc4af1ea405e674ba20 is 50, key is test_row_0/C:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:03,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742199_1375 (size=12301) 2024-12-11T06:16:03,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897823538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897823538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897823538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897823549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897823549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:03,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:03,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/eba4145e716f4dc4af1ea405e674ba20 2024-12-11T06:16:03,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/e960dabb066c4a2ead3147dc055bc227 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227 2024-12-11T06:16:03,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227, entries=200, sequenceid=298, filesize=39.0 K 2024-12-11T06:16:03,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/109c65b2d6c344249e27aa6da69c7cf9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9 2024-12-11T06:16:03,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9, entries=150, sequenceid=298, filesize=12.0 K 2024-12-11T06:16:03,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/eba4145e716f4dc4af1ea405e674ba20 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20 2024-12-11T06:16:03,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20, entries=150, sequenceid=298, filesize=12.0 K 2024-12-11T06:16:03,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~209.30 KB/214320, currentSize=107.34 KB/109920 for 4087fec5b081d47691e33f5c7c3537f9 in 949ms, sequenceid=298, compaction requested=true 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:03,760 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:03,760 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:03,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:03,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128429 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:03,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:16:03,761 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:03,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,761 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:16:03,761 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=125.4 K 2024-12-11T06:16:03,761 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/6553abe7abe64735b9eebf2b35eb4b42, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.1 K 2024-12-11T06:16:03,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227] 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6553abe7abe64735b9eebf2b35eb4b42, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting df01b9cf155b4256a9c4fe3ef3044c3b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e8393f951541348e960f6a2971faa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733897761680 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf01b4bf4a264efe9eefa0ccf6b83486, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733897760487 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 109c65b2d6c344249e27aa6da69c7cf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:03,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e960dabb066c4a2ead3147dc055bc227, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:03,768 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:03,769 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/1cdebe7789a242e9accdfc124a9bc776 is 50, key is test_row_0/B:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:03,770 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:03,774 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121122063e464ad445f6b93658a523e2ac18_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:03,777 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121122063e464ad445f6b93658a523e2ac18_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:03,778 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121122063e464ad445f6b93658a523e2ac18_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:03,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742200_1376 (size=12745) 2024-12-11T06:16:03,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742201_1377 (size=4469) 2024-12-11T06:16:03,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T06:16:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:03,819 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211aa8c608e8ab949c299e66e3aefa162e0_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897762884/Put/seqid=0 2024-12-11T06:16:03,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742202_1378 (size=12454) 2024-12-11T06:16:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897824085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897824085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897824094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,184 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/1cdebe7789a242e9accdfc124a9bc776 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/1cdebe7789a242e9accdfc124a9bc776 2024-12-11T06:16:04,187 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:04,188 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c7f281376cc45f7b58bdab8874c592b is 175, key is test_row_0/A:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:04,188 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into 1cdebe7789a242e9accdfc124a9bc776(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:04,188 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:04,188 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897763760; duration=0sec 2024-12-11T06:16:04,189 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:04,189 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:16:04,189 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:04,189 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:04,189 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:16:04,190 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:04,190 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/a5ef90d7dd76463882696edc22fd5d61, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.1 K 2024-12-11T06:16:04,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ef90d7dd76463882696edc22fd5d61, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733897759995 2024-12-11T06:16:04,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 465f1c8d1e184926aeb8251a3e8e7ac7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733897761680 2024-12-11T06:16:04,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting eba4145e716f4dc4af1ea405e674ba20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:04,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742203_1379 (size=31699) 2024-12-11T06:16:04,195 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8c7f281376cc45f7b58bdab8874c592b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b 2024-12-11T06:16:04,198 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:04,199 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into 8c7f281376cc45f7b58bdab8874c592b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:04,199 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:04,199 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897763760; duration=0sec 2024-12-11T06:16:04,199 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:04,199 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:16:04,199 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/dcc7f8e3d74a41858cadd26e05fc2b6f is 50, key is test_row_0/C:col10/1733897762809/Put/seqid=0 2024-12-11T06:16:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742204_1380 (size=12745) 2024-12-11T06:16:04,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897824198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897824198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897824209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:04,235 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211aa8c608e8ab949c299e66e3aefa162e0_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211aa8c608e8ab949c299e66e3aefa162e0_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:04,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/91decef271b74c1c9a77d40bdd95cf20, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:04,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/91decef271b74c1c9a77d40bdd95cf20 is 175, key is test_row_0/A:col10/1733897762884/Put/seqid=0 2024-12-11T06:16:04,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742205_1381 (size=31255) 2024-12-11T06:16:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897824409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897824409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897824418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897824568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897824568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,572 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,573 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:04,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,606 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/dcc7f8e3d74a41858cadd26e05fc2b6f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/dcc7f8e3d74a41858cadd26e05fc2b6f 2024-12-11T06:16:04,609 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into dcc7f8e3d74a41858cadd26e05fc2b6f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:04,609 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:04,609 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897763760; duration=0sec 2024-12-11T06:16:04,610 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:04,610 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:16:04,640 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/91decef271b74c1c9a77d40bdd95cf20 2024-12-11T06:16:04,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2f3eb5344814456ebf7eabb5cf387efe is 50, key is test_row_0/B:col10/1733897762884/Put/seqid=0 2024-12-11T06:16:04,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742206_1382 (size=12301) 2024-12-11T06:16:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897824719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897824719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:04,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897824728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,075 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2f3eb5344814456ebf7eabb5cf387efe 2024-12-11T06:16:05,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/811af9365a60473f8e12f60f0b72bd9f is 50, key is test_row_0/C:col10/1733897762884/Put/seqid=0 2024-12-11T06:16:05,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742207_1383 (size=12301) 2024-12-11T06:16:05,087 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/811af9365a60473f8e12f60f0b72bd9f 2024-12-11T06:16:05,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/91decef271b74c1c9a77d40bdd95cf20 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20 2024-12-11T06:16:05,093 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20, entries=150, sequenceid=317, filesize=30.5 K 2024-12-11T06:16:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2f3eb5344814456ebf7eabb5cf387efe as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe 2024-12-11T06:16:05,097 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe, entries=150, sequenceid=317, filesize=12.0 K 2024-12-11T06:16:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/811af9365a60473f8e12f60f0b72bd9f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f 2024-12-11T06:16:05,101 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f, entries=150, sequenceid=317, filesize=12.0 K 2024-12-11T06:16:05,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4087fec5b081d47691e33f5c7c3537f9 in 1283ms, sequenceid=317, compaction requested=false 2024-12-11T06:16:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-11T06:16:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-11T06:16:05,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-11T06:16:05,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3200 sec 2024-12-11T06:16:05,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.3270 sec 2024-12-11T06:16:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:05,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:16:05,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:05,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:05,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:05,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:05,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e9a5f775d8394bbd8016edf0c10e91d1_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:05,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742208_1384 (size=17534) 2024-12-11T06:16:05,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:05,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e9a5f775d8394bbd8016edf0c10e91d1_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e9a5f775d8394bbd8016edf0c10e91d1_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:05,266 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/026d3f7014d9423eb18537ec2fcd06c8, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:05,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/026d3f7014d9423eb18537ec2fcd06c8 is 175, key is test_row_0/A:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742209_1385 (size=48639) 2024-12-11T06:16:05,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897825292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897825301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897825305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897825408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897825418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897825418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897825619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897825628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897825628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/026d3f7014d9423eb18537ec2fcd06c8 2024-12-11T06:16:05,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/298013e499c34bc7b1924294ad005f6b is 50, key is test_row_0/B:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:05,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742210_1386 (size=12301) 2024-12-11T06:16:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897825928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897825938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T06:16:05,949 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:05,950 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897825939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-11T06:16:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:05,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:05,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:05,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:06,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/298013e499c34bc7b1924294ad005f6b 2024-12-11T06:16:06,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/8d8bc15318f54cefa078b7cab2c33a78 is 50, key is test_row_0/C:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:06,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742211_1387 (size=12301) 2024-12-11T06:16:06,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/8d8bc15318f54cefa078b7cab2c33a78 2024-12-11T06:16:06,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/026d3f7014d9423eb18537ec2fcd06c8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8 2024-12-11T06:16:06,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8, entries=250, sequenceid=335, filesize=47.5 K 2024-12-11T06:16:06,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/298013e499c34bc7b1924294ad005f6b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b 2024-12-11T06:16:06,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T06:16:06,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:06,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:06,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:06,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b, entries=150, sequenceid=335, filesize=12.0 K 2024-12-11T06:16:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/8d8bc15318f54cefa078b7cab2c33a78 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78 2024-12-11T06:16:06,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78, entries=150, sequenceid=335, filesize=12.0 K 2024-12-11T06:16:06,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4087fec5b081d47691e33f5c7c3537f9 in 882ms, sequenceid=335, compaction requested=true 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:06,117 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:06,117 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:06,118 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:06,118 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:06,118 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:16:06,118 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:16:06,118 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,118 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,118 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/1cdebe7789a242e9accdfc124a9bc776, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.5 K 2024-12-11T06:16:06,118 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=109.0 K 2024-12-11T06:16:06,118 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,118 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8] 2024-12-11T06:16:06,119 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdebe7789a242e9accdfc124a9bc776, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:06,119 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7f281376cc45f7b58bdab8874c592b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:06,119 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3eb5344814456ebf7eabb5cf387efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733897762852 2024-12-11T06:16:06,119 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91decef271b74c1c9a77d40bdd95cf20, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733897762852 2024-12-11T06:16:06,119 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 298013e499c34bc7b1924294ad005f6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:06,121 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 026d3f7014d9423eb18537ec2fcd06c8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:06,128 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:06,128 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/127dbfe9c6fd45b3a1074c88fb5c377c is 50, key is test_row_0/B:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:06,129 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:06,132 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211a3edf167e4784a3c97871fab04a098e1_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:06,133 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211a3edf167e4784a3c97871fab04a098e1_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:06,134 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a3edf167e4784a3c97871fab04a098e1_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742212_1388 (size=12847) 2024-12-11T06:16:06,144 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/127dbfe9c6fd45b3a1074c88fb5c377c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/127dbfe9c6fd45b3a1074c88fb5c377c 2024-12-11T06:16:06,148 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into 127dbfe9c6fd45b3a1074c88fb5c377c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:06,148 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:06,148 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897766117; duration=0sec 2024-12-11T06:16:06,148 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:06,148 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:16:06,148 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:06,154 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:06,154 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:16:06,154 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742213_1389 (size=4469) 2024-12-11T06:16:06,154 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/dcc7f8e3d74a41858cadd26e05fc2b6f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.5 K 2024-12-11T06:16:06,155 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc7f8e3d74a41858cadd26e05fc2b6f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733897762454 2024-12-11T06:16:06,155 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 811af9365a60473f8e12f60f0b72bd9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733897762852 2024-12-11T06:16:06,155 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8bc15318f54cefa078b7cab2c33a78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:06,160 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:06,160 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c83da71e8c9a4faf8fdfed8c13a94a51 is 50, key is test_row_0/C:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:06,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742214_1390 (size=12847) 2024-12-11T06:16:06,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:06,259 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110e14253629f341e0a9b5ec3ffa6aaef4_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897765295/Put/seqid=0 2024-12-11T06:16:06,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742215_1391 (size=12454) 2024-12-11T06:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:06,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897826483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897826483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897826489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,555 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:06,556 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f656206dad8643139de8b3e81245aa41 is 175, key is test_row_0/A:col10/1733897765233/Put/seqid=0 2024-12-11T06:16:06,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742216_1392 (size=31801) 2024-12-11T06:16:06,568 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/f656206dad8643139de8b3e81245aa41 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41 2024-12-11T06:16:06,572 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c83da71e8c9a4faf8fdfed8c13a94a51 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c83da71e8c9a4faf8fdfed8c13a94a51 2024-12-11T06:16:06,576 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into f656206dad8643139de8b3e81245aa41(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:06,576 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:06,576 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897766117; duration=0sec 2024-12-11T06:16:06,576 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:06,576 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:16:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:06,588 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into c83da71e8c9a4faf8fdfed8c13a94a51(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:06,588 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:06,588 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897766117; duration=0sec 2024-12-11T06:16:06,588 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:06,588 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:16:06,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897826589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897826589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,594 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,594 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 4087fec5b081d47691e33f5c7c3537f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:06,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897826599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897826599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897826608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:06,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110e14253629f341e0a9b5ec3ffa6aaef4_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110e14253629f341e0a9b5ec3ffa6aaef4_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/496b58ea7aca489fb1cf1adc1cc4d09d, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/496b58ea7aca489fb1cf1adc1cc4d09d is 175, key is test_row_0/A:col10/1733897765295/Put/seqid=0 2024-12-11T06:16:06,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742217_1393 (size=31255) 2024-12-11T06:16:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897826808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897826819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:06,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897826828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:07,123 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/496b58ea7aca489fb1cf1adc1cc4d09d 2024-12-11T06:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bcd9740598ff4791854d47bbb52437d8 is 50, key is test_row_0/B:col10/1733897765295/Put/seqid=0 2024-12-11T06:16:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897827129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897827129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897827143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742218_1394 (size=12301) 2024-12-11T06:16:07,172 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bcd9740598ff4791854d47bbb52437d8 2024-12-11T06:16:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c85f808bcc534814bbcc3401d39a1eff is 50, key is test_row_0/C:col10/1733897765295/Put/seqid=0 2024-12-11T06:16:07,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742219_1395 (size=12301) 2024-12-11T06:16:07,600 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c85f808bcc534814bbcc3401d39a1eff 2024-12-11T06:16:07,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/496b58ea7aca489fb1cf1adc1cc4d09d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d 2024-12-11T06:16:07,607 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d, entries=150, sequenceid=353, filesize=30.5 K 2024-12-11T06:16:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bcd9740598ff4791854d47bbb52437d8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8 2024-12-11T06:16:07,610 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8, entries=150, sequenceid=353, filesize=12.0 K 2024-12-11T06:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/c85f808bcc534814bbcc3401d39a1eff as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff 2024-12-11T06:16:07,614 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff, entries=150, sequenceid=353, filesize=12.0 K 2024-12-11T06:16:07,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4087fec5b081d47691e33f5c7c3537f9 in 1356ms, sequenceid=353, compaction requested=false 2024-12-11T06:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-11T06:16:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-11T06:16:07,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-11T06:16:07,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6650 sec 2024-12-11T06:16:07,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6710 sec 2024-12-11T06:16:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:07,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:07,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:07,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a0a28d7bea1e4ad69bd0a989ae84d124_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:07,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742220_1396 (size=14994) 2024-12-11T06:16:07,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897827684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897827693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897827693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897827799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897827808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897827808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897828009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897828030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897828033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:08,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a0a28d7bea1e4ad69bd0a989ae84d124_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0a28d7bea1e4ad69bd0a989ae84d124_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:08,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b0ab29f7803142d193196290ea435aeb, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:08,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b0ab29f7803142d193196290ea435aeb is 175, key is test_row_0/A:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:08,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742221_1397 (size=39949) 2024-12-11T06:16:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T06:16:08,100 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:08,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-11T06:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:08,105 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:08,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:08,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:08,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:08,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897828329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897828348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897828348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:08,478 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b0ab29f7803142d193196290ea435aeb 2024-12-11T06:16:08,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2337a6408c9f40188dfd953ec3714056 is 50, key is test_row_0/B:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:08,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742222_1398 (size=12301) 2024-12-11T06:16:08,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:08,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:08,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897828848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:08,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897828868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897828868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:08,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2337a6408c9f40188dfd953ec3714056 2024-12-11T06:16:08,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ede892bfff5a463ca775d1f4e30e7401 is 50, key is test_row_0/C:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:08,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742223_1399 (size=12301) 2024-12-11T06:16:09,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:09,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:09,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ede892bfff5a463ca775d1f4e30e7401 2024-12-11T06:16:09,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/b0ab29f7803142d193196290ea435aeb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb 2024-12-11T06:16:09,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb, entries=200, sequenceid=371, filesize=39.0 K 2024-12-11T06:16:09,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2337a6408c9f40188dfd953ec3714056 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056 2024-12-11T06:16:09,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056, entries=150, sequenceid=371, filesize=12.0 K 2024-12-11T06:16:09,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ede892bfff5a463ca775d1f4e30e7401 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401 2024-12-11T06:16:09,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401, entries=150, sequenceid=371, filesize=12.0 K 2024-12-11T06:16:09,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 4087fec5b081d47691e33f5c7c3537f9 in 1672ms, sequenceid=371, compaction requested=true 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:09,325 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:09,325 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:16:09,326 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,326 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,326 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/127dbfe9c6fd45b3a1074c88fb5c377c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.6 K 2024-12-11T06:16:09,326 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=100.6 K 2024-12-11T06:16:09,326 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb] 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 127dbfe9c6fd45b3a1074c88fb5c377c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:09,326 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f656206dad8643139de8b3e81245aa41, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:09,327 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd9740598ff4791854d47bbb52437d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733897765290 2024-12-11T06:16:09,327 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 496b58ea7aca489fb1cf1adc1cc4d09d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733897765290 2024-12-11T06:16:09,327 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2337a6408c9f40188dfd953ec3714056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766482 2024-12-11T06:16:09,327 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ab29f7803142d193196290ea435aeb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766480 2024-12-11T06:16:09,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T06:16:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:09,332 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:09,334 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:09,334 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bb604c8c3c594f5fb59cd2f07e24f559 is 50, key is test_row_0/B:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:09,345 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412113d2aaa9ed1274685b5763ed56e2341a7_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:09,347 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412113d2aaa9ed1274685b5763ed56e2341a7_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:09,347 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113d2aaa9ed1274685b5763ed56e2341a7_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:09,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e15c44b409764e4f9e8c7cdd3e50e28b_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897767691/Put/seqid=0 2024-12-11T06:16:09,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742224_1400 (size=12949) 2024-12-11T06:16:09,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742225_1401 (size=4469) 2024-12-11T06:16:09,402 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#336 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:09,403 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/74bb10e04e0a40fab98f38d5956e7e76 is 175, key is test_row_0/A:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:09,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742226_1402 (size=12454) 2024-12-11T06:16:09,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:09,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742227_1403 (size=31903) 2024-12-11T06:16:09,433 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e15c44b409764e4f9e8c7cdd3e50e28b_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e15c44b409764e4f9e8c7cdd3e50e28b_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/65bd98da51cb4a16a7efaa5b43df5663, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:09,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/65bd98da51cb4a16a7efaa5b43df5663 is 175, key is test_row_0/A:col10/1733897767691/Put/seqid=0 2024-12-11T06:16:09,440 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/74bb10e04e0a40fab98f38d5956e7e76 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76 2024-12-11T06:16:09,446 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into 74bb10e04e0a40fab98f38d5956e7e76(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:09,446 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:09,446 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897769325; duration=0sec 2024-12-11T06:16:09,446 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:09,446 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:16:09,446 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:09,447 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:09,447 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:16:09,447 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:09,447 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c83da71e8c9a4faf8fdfed8c13a94a51, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.6 K 2024-12-11T06:16:09,448 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83da71e8c9a4faf8fdfed8c13a94a51, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733897764073 2024-12-11T06:16:09,448 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85f808bcc534814bbcc3401d39a1eff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733897765290 2024-12-11T06:16:09,448 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede892bfff5a463ca775d1f4e30e7401, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766482 2024-12-11T06:16:09,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742228_1404 (size=31255) 2024-12-11T06:16:09,481 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:09,482 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/800bc8c27d3c4a2ea3a23b109b55dde5 is 50, key is test_row_0/C:col10/1733897766487/Put/seqid=0 2024-12-11T06:16:09,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742229_1405 (size=12949) 2024-12-11T06:16:09,761 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bb604c8c3c594f5fb59cd2f07e24f559 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb604c8c3c594f5fb59cd2f07e24f559 2024-12-11T06:16:09,767 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into bb604c8c3c594f5fb59cd2f07e24f559(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:09,767 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:09,767 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897769325; duration=0sec 2024-12-11T06:16:09,767 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:09,767 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:16:09,868 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/65bd98da51cb4a16a7efaa5b43df5663 2024-12-11T06:16:09,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/8060fa57e9d24e72af66049b17e6dd97 is 50, key is test_row_0/B:col10/1733897767691/Put/seqid=0 2024-12-11T06:16:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:09,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. as already flushing 2024-12-11T06:16:09,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742230_1406 (size=12301) 2024-12-11T06:16:09,920 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/800bc8c27d3c4a2ea3a23b109b55dde5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/800bc8c27d3c4a2ea3a23b109b55dde5 2024-12-11T06:16:09,933 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into 800bc8c27d3c4a2ea3a23b109b55dde5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:09,933 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:09,933 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897769325; duration=0sec 2024-12-11T06:16:09,933 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:09,933 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:16:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897829958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:09,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897829960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:09,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897829960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897830080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897830080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897830080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897830299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897830299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897830309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,313 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/8060fa57e9d24e72af66049b17e6dd97 2024-12-11T06:16:10,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ebfc6679684543e594cc563e01f5e4f9 is 50, key is test_row_0/C:col10/1733897767691/Put/seqid=0 2024-12-11T06:16:10,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742231_1407 (size=12301) 2024-12-11T06:16:10,374 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ebfc6679684543e594cc563e01f5e4f9 2024-12-11T06:16:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/65bd98da51cb4a16a7efaa5b43df5663 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663 2024-12-11T06:16:10,387 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663, entries=150, sequenceid=388, filesize=30.5 K 2024-12-11T06:16:10,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/8060fa57e9d24e72af66049b17e6dd97 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97 2024-12-11T06:16:10,393 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97, entries=150, sequenceid=388, filesize=12.0 K 2024-12-11T06:16:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/ebfc6679684543e594cc563e01f5e4f9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9 2024-12-11T06:16:10,414 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9, entries=150, sequenceid=388, filesize=12.0 K 2024-12-11T06:16:10,414 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4087fec5b081d47691e33f5c7c3537f9 in 1083ms, sequenceid=388, compaction requested=false 2024-12-11T06:16:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-11T06:16:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-11T06:16:10,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-11T06:16:10,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3150 sec 2024-12-11T06:16:10,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3240 sec 2024-12-11T06:16:10,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:10,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:10,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e39828cfd21d45029eb51b675eeff80b_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:10,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897830637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742232_1408 (size=12454) 2024-12-11T06:16:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897830644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897830640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897830648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897830649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e39828cfd21d45029eb51b675eeff80b_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e39828cfd21d45029eb51b675eeff80b_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:10,686 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/c616de750e074bc7bf4ae847615191a7, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:10,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/c616de750e074bc7bf4ae847615191a7 is 175, key is test_row_0/A:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742233_1409 (size=31255) 2024-12-11T06:16:10,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/c616de750e074bc7bf4ae847615191a7 2024-12-11T06:16:10,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2b15d6f58355467281e8838a8494254e is 50, key is test_row_0/B:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:10,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897830759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742234_1410 (size=12301) 2024-12-11T06:16:10,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897830770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897830772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897830780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897830780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,963 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-12-11T06:16:10,963 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:10,963 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,963 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,963 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897830969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,973 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-12-11T06:16:10,973 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-12-11T06:16:10,973 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:10,973 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:10,973 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,973 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,973 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,973 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:10,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:10,975 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-12-11T06:16:10,975 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:10,975 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,975 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:10,985 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-12-11T06:16:10,985 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:10,986 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,986 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:10,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897830988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897830988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897830998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897830998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2b15d6f58355467281e8838a8494254e 2024-12-11T06:16:11,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/04de3a495a474337bd8725b00b276477 is 50, key is test_row_0/C:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:11,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742235_1411 (size=12301) 2024-12-11T06:16:11,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60726 deadline: 1733897831279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:11,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60728 deadline: 1733897831298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60694 deadline: 1733897831298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60668 deadline: 1733897831309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60760 deadline: 1733897831309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:11,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/04de3a495a474337bd8725b00b276477 2024-12-11T06:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/c616de750e074bc7bf4ae847615191a7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7 2024-12-11T06:16:11,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7, entries=150, sequenceid=409, filesize=30.5 K 2024-12-11T06:16:11,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/2b15d6f58355467281e8838a8494254e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e 2024-12-11T06:16:11,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e, entries=150, sequenceid=409, filesize=12.0 K 2024-12-11T06:16:11,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/04de3a495a474337bd8725b00b276477 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477 2024-12-11T06:16:11,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477, entries=150, sequenceid=409, filesize=12.0 K 2024-12-11T06:16:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4087fec5b081d47691e33f5c7c3537f9 in 985ms, sequenceid=409, compaction requested=true 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:11,598 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:11,598 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4087fec5b081d47691e33f5c7c3537f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/A is initiating minor compaction (all files) 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/B is initiating minor compaction (all files) 2024-12-11T06:16:11,599 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/A in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:11,599 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/B in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:11,599 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=92.2 K 2024-12-11T06:16:11,599 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb604c8c3c594f5fb59cd2f07e24f559, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.7 K 2024-12-11T06:16:11,599 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7] 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb604c8c3c594f5fb59cd2f07e24f559, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766482 2024-12-11T06:16:11,599 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bb10e04e0a40fab98f38d5956e7e76, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766482 2024-12-11T06:16:11,600 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8060fa57e9d24e72af66049b17e6dd97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733897767675 2024-12-11T06:16:11,600 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 65bd98da51cb4a16a7efaa5b43df5663, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733897767675 2024-12-11T06:16:11,600 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c616de750e074bc7bf4ae847615191a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733897769955 2024-12-11T06:16:11,600 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b15d6f58355467281e8838a8494254e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733897769955 2024-12-11T06:16:11,607 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:11,607 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/9e3e932a567343aaa3d67d0ed7c05948 is 50, key is test_row_0/B:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:11,609 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:11,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742236_1412 (size=13051) 2024-12-11T06:16:11,611 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211b0609352ec7244f688822682e2f04e13_4087fec5b081d47691e33f5c7c3537f9 store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:11,614 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/9e3e932a567343aaa3d67d0ed7c05948 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9e3e932a567343aaa3d67d0ed7c05948 2024-12-11T06:16:11,618 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/B of 4087fec5b081d47691e33f5c7c3537f9 into 9e3e932a567343aaa3d67d0ed7c05948(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:11,618 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:11,619 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/B, priority=13, startTime=1733897771598; duration=0sec 2024-12-11T06:16:11,619 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:11,619 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:B 2024-12-11T06:16:11,619 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:11,620 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:11,620 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 4087fec5b081d47691e33f5c7c3537f9/C is initiating minor compaction (all files) 2024-12-11T06:16:11,620 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4087fec5b081d47691e33f5c7c3537f9/C in TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:11,620 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/800bc8c27d3c4a2ea3a23b109b55dde5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp, totalSize=36.7 K 2024-12-11T06:16:11,620 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800bc8c27d3c4a2ea3a23b109b55dde5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733897766482 2024-12-11T06:16:11,621 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfc6679684543e594cc563e01f5e4f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733897767675 2024-12-11T06:16:11,621 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04de3a495a474337bd8725b00b276477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733897769955 2024-12-11T06:16:11,626 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211b0609352ec7244f688822682e2f04e13_4087fec5b081d47691e33f5c7c3537f9, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:11,626 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b0609352ec7244f688822682e2f04e13_4087fec5b081d47691e33f5c7c3537f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:11,629 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:11,629 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/1d453342cf0e46a1bdb8869d7cd43cd4 is 50, key is test_row_0/C:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742237_1413 (size=4469) 2024-12-11T06:16:11,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742238_1414 (size=13051) 2024-12-11T06:16:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:11,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:11,790 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:11,790 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:11,790 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:11,790 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:11,832 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-12-11T06:16:11,832 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-12-11T06:16:11,832 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:11,832 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:11,832 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-12-11T06:16:11,832 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:11,832 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:11,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:11,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:11,834 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-12-11T06:16:11,834 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:11,834 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,834 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:11,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:11,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121190a43ef4e9804060a95515fe1248d5d5_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897770645/Put/seqid=0 2024-12-11T06:16:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742239_1415 (size=12454) 2024-12-11T06:16:12,037 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4087fec5b081d47691e33f5c7c3537f9#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:12,037 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8169764e243a49729f061d1afa089249 is 175, key is test_row_0/A:col10/1733897770611/Put/seqid=0 2024-12-11T06:16:12,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742240_1416 (size=32005) 2024-12-11T06:16:12,042 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/1d453342cf0e46a1bdb8869d7cd43cd4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/1d453342cf0e46a1bdb8869d7cd43cd4 2024-12-11T06:16:12,046 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/C of 4087fec5b081d47691e33f5c7c3537f9 into 1d453342cf0e46a1bdb8869d7cd43cd4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:12,046 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:12,046 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/C, priority=13, startTime=1733897771598; duration=0sec 2024-12-11T06:16:12,046 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:12,046 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:C 2024-12-11T06:16:12,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:12,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121190a43ef4e9804060a95515fe1248d5d5_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121190a43ef4e9804060a95515fe1248d5d5_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:12,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bbe1e8154d06448c9c8e2b729b3b2ec9, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:12,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bbe1e8154d06448c9c8e2b729b3b2ec9 is 175, key is test_row_0/A:col10/1733897770645/Put/seqid=0 2024-12-11T06:16:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742241_1417 (size=31255) 2024-12-11T06:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T06:16:12,259 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1006 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3018 rows 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1014 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3039 rows 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1007 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3020 rows 2024-12-11T06:16:12,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1012 2024-12-11T06:16:12,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3036 rows 2024-12-11T06:16:12,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1022 2024-12-11T06:16:12,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3064 rows 2024-12-11T06:16:12,260 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:16:12,260 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:16:12,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:12,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:12,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:12,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:16:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:12,263 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897772263"}]},"ts":"1733897772263"} 2024-12-11T06:16:12,264 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:16:12,264 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:16:12,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:16:12,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, UNASSIGN}] 2024-12-11T06:16:12,267 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, UNASSIGN 2024-12-11T06:16:12,268 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:12,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, UNASSIGN because future has completed 2024-12-11T06:16:12,269 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:16:12,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:12,421 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:12,421 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:16:12,422 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 4087fec5b081d47691e33f5c7c3537f9, disabling compactions & flushes 2024-12-11T06:16:12,422 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 1 compactions & cache flush to complete for region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:12,449 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/8169764e243a49729f061d1afa089249 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8169764e243a49729f061d1afa089249 2024-12-11T06:16:12,456 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4087fec5b081d47691e33f5c7c3537f9/A of 4087fec5b081d47691e33f5c7c3537f9 into 8169764e243a49729f061d1afa089249(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:12,456 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:12,456 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:12,456 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., storeName=4087fec5b081d47691e33f5c7c3537f9/A, priority=13, startTime=1733897771598; duration=0sec 2024-12-11T06:16:12,457 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:12,457 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4087fec5b081d47691e33f5c7c3537f9:A 2024-12-11T06:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:12,651 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bbe1e8154d06448c9c8e2b729b3b2ec9 2024-12-11T06:16:12,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bb5d268936764733aac5b85dddfb6170 is 50, key is test_row_0/B:col10/1733897770645/Put/seqid=0 2024-12-11T06:16:12,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742242_1418 (size=12301) 2024-12-11T06:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:13,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bb5d268936764733aac5b85dddfb6170 2024-12-11T06:16:13,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/b4fa8cebd5274597bb6b220e5a7a88bd is 50, key is test_row_0/C:col10/1733897770645/Put/seqid=0 2024-12-11T06:16:13,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742243_1419 (size=12301) 2024-12-11T06:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:13,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/b4fa8cebd5274597bb6b220e5a7a88bd 2024-12-11T06:16:13,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/bbe1e8154d06448c9c8e2b729b3b2ec9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bbe1e8154d06448c9c8e2b729b3b2ec9 2024-12-11T06:16:13,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bbe1e8154d06448c9c8e2b729b3b2ec9, entries=150, sequenceid=426, filesize=30.5 K 2024-12-11T06:16:13,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/bb5d268936764733aac5b85dddfb6170 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb5d268936764733aac5b85dddfb6170 2024-12-11T06:16:13,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb5d268936764733aac5b85dddfb6170, entries=150, sequenceid=426, filesize=12.0 K 2024-12-11T06:16:13,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/b4fa8cebd5274597bb6b220e5a7a88bd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/b4fa8cebd5274597bb6b220e5a7a88bd 2024-12-11T06:16:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/b4fa8cebd5274597bb6b220e5a7a88bd, entries=150, sequenceid=426, filesize=12.0 K 2024-12-11T06:16:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 4087fec5b081d47691e33f5c7c3537f9 in 1705ms, sequenceid=426, compaction requested=false 2024-12-11T06:16:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4087fec5b081d47691e33f5c7c3537f9: 2024-12-11T06:16:13,496 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. after waiting 0 ms 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:13,496 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 4087fec5b081d47691e33f5c7c3537f9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=A 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=B 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4087fec5b081d47691e33f5c7c3537f9, store=C 2024-12-11T06:16:13,496 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:13,502 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121131e1f6ca9a1c46c1ae69ea41d322ad5a_4087fec5b081d47691e33f5c7c3537f9 is 50, key is test_row_0/A:col10/1733897771819/Put/seqid=0 2024-12-11T06:16:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742244_1420 (size=12454) 2024-12-11T06:16:13,906 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:13,912 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121131e1f6ca9a1c46c1ae69ea41d322ad5a_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131e1f6ca9a1c46c1ae69ea41d322ad5a_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:13,913 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/be6dea4021dc42ab9b5bc2ee4f641f8b, store: [table=TestAcidGuarantees family=A region=4087fec5b081d47691e33f5c7c3537f9] 2024-12-11T06:16:13,914 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/be6dea4021dc42ab9b5bc2ee4f641f8b is 175, key is test_row_0/A:col10/1733897771819/Put/seqid=0 2024-12-11T06:16:13,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742245_1421 (size=31255) 2024-12-11T06:16:14,321 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/be6dea4021dc42ab9b5bc2ee4f641f8b 2024-12-11T06:16:14,326 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0186a11844484a6f8be197e1df6e2755 is 50, key is test_row_0/B:col10/1733897771819/Put/seqid=0 2024-12-11T06:16:14,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742246_1422 (size=12301) 2024-12-11T06:16:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:14,730 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0186a11844484a6f8be197e1df6e2755 2024-12-11T06:16:14,738 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/7e3c2bffffcd44eeba09fbdde8e3b09a is 50, key is test_row_0/C:col10/1733897771819/Put/seqid=0 2024-12-11T06:16:14,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742247_1423 (size=12301) 2024-12-11T06:16:15,143 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/7e3c2bffffcd44eeba09fbdde8e3b09a 2024-12-11T06:16:15,148 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/A/be6dea4021dc42ab9b5bc2ee4f641f8b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/be6dea4021dc42ab9b5bc2ee4f641f8b 2024-12-11T06:16:15,151 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/be6dea4021dc42ab9b5bc2ee4f641f8b, entries=150, sequenceid=435, filesize=30.5 K 2024-12-11T06:16:15,151 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/B/0186a11844484a6f8be197e1df6e2755 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0186a11844484a6f8be197e1df6e2755 2024-12-11T06:16:15,155 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0186a11844484a6f8be197e1df6e2755, entries=150, sequenceid=435, filesize=12.0 K 2024-12-11T06:16:15,155 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/.tmp/C/7e3c2bffffcd44eeba09fbdde8e3b09a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/7e3c2bffffcd44eeba09fbdde8e3b09a 2024-12-11T06:16:15,160 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/7e3c2bffffcd44eeba09fbdde8e3b09a, entries=150, sequenceid=435, filesize=12.0 K 2024-12-11T06:16:15,161 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4087fec5b081d47691e33f5c7c3537f9 in 1664ms, sequenceid=435, compaction requested=true 2024-12-11T06:16:15,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7] to archive 2024-12-11T06:16:15,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/ab6984b2fecc4c0d8bfc3d3da6a392ad 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/aa697565116f4b1c99605f2411998ba6 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b9e6abd78f2f4f838335482215ec4492 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/96fc17b5914841748cb3451ba27a021e 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/51a8d947f0e442fe982bdb718ed8fe8d 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f46b88458308417b845fa10abd1f3e89 2024-12-11T06:16:15,166 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/7a79663f8b9940a29a7b5b1e292f605f 2024-12-11T06:16:15,168 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/9fbf9e08067c4e60b32f245535c4078d 2024-12-11T06:16:15,168 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/624f16b9a39944cdb915dae88f3e9341 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/e960dabb066c4a2ead3147dc055bc227 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/91decef271b74c1c9a77d40bdd95cf20 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c7f281376cc45f7b58bdab8874c592b 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8c62d9eeca4b4e1e88126de81a055dd2 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bf01b4bf4a264efe9eefa0ccf6b83486 2024-12-11T06:16:15,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/df01b9cf155b4256a9c4fe3ef3044c3b 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/026d3f7014d9423eb18537ec2fcd06c8 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/f656206dad8643139de8b3e81245aa41 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/496b58ea7aca489fb1cf1adc1cc4d09d 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/b0ab29f7803142d193196290ea435aeb 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/c616de750e074bc7bf4ae847615191a7 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/65bd98da51cb4a16a7efaa5b43df5663 2024-12-11T06:16:15,171 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/74bb10e04e0a40fab98f38d5956e7e76 2024-12-11T06:16:15,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e7026008f02e4e6a9c3fa3fe669f75f9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ccbd2fa72f4a414a913a693685cf480f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/6553abe7abe64735b9eebf2b35eb4b42, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/1cdebe7789a242e9accdfc124a9bc776, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/127dbfe9c6fd45b3a1074c88fb5c377c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb604c8c3c594f5fb59cd2f07e24f559, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e] to archive 2024-12-11T06:16:15,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:15,179 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0d356847bc854829a2a50be55828ee81 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/03590cdf02774938b16c9103efd86bc2 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e7026008f02e4e6a9c3fa3fe669f75f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e7026008f02e4e6a9c3fa3fe669f75f9 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/e5dc55c7252346ab9e48e1616b94a317 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ccbd2fa72f4a414a913a693685cf480f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ccbd2fa72f4a414a913a693685cf480f 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/027f4baa9f484c3a96d43d5ab740226a 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/20acd5ca02f74e35a56517307600678d 2024-12-11T06:16:15,180 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9d22160c7bc2430e89dc9a3eafa5629b 2024-12-11T06:16:15,181 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/96dec828976448549d8c69ae229aa73e 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/55e8393f951541348e960f6a2971faa9 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/ce24139179fb466e88057bd71bd8c549 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/6553abe7abe64735b9eebf2b35eb4b42 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/6553abe7abe64735b9eebf2b35eb4b42 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/109c65b2d6c344249e27aa6da69c7cf9 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/1cdebe7789a242e9accdfc124a9bc776 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/1cdebe7789a242e9accdfc124a9bc776 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2f3eb5344814456ebf7eabb5cf387efe 2024-12-11T06:16:15,182 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/127dbfe9c6fd45b3a1074c88fb5c377c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/127dbfe9c6fd45b3a1074c88fb5c377c 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb604c8c3c594f5fb59cd2f07e24f559 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb604c8c3c594f5fb59cd2f07e24f559 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bcd9740598ff4791854d47bbb52437d8 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/298013e499c34bc7b1924294ad005f6b 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/8060fa57e9d24e72af66049b17e6dd97 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2b15d6f58355467281e8838a8494254e 2024-12-11T06:16:15,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/2337a6408c9f40188dfd953ec3714056 2024-12-11T06:16:15,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/d51a5cd2343c426dbfef54545bbb7589, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/108a6e2ea4da4d23837a9c79d38ef28d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/a5ef90d7dd76463882696edc22fd5d61, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/dcc7f8e3d74a41858cadd26e05fc2b6f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c83da71e8c9a4faf8fdfed8c13a94a51, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/800bc8c27d3c4a2ea3a23b109b55dde5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477] to archive 2024-12-11T06:16:15,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/d51a5cd2343c426dbfef54545bbb7589 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/d51a5cd2343c426dbfef54545bbb7589 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/21ae86bb050f4b7493b4b450a0f9b65c 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/da7698d6ae994abc905c159534132874 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d6d45800bc14325878d1dd91475a69a 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/292a58969f8a4020b5ac2b03f70e40bc 2024-12-11T06:16:15,189 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/9d723c787c4b452fb1946c12b6e7b33c 2024-12-11T06:16:15,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/813b0cf1c47a4fe992b5f6b41e8348f0 2024-12-11T06:16:15,190 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c8d401790a0b4711a0aa57602b191c58 2024-12-11T06:16:15,190 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/108a6e2ea4da4d23837a9c79d38ef28d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/108a6e2ea4da4d23837a9c79d38ef28d 2024-12-11T06:16:15,191 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/a5ef90d7dd76463882696edc22fd5d61 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/a5ef90d7dd76463882696edc22fd5d61 2024-12-11T06:16:15,192 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/465f1c8d1e184926aeb8251a3e8e7ac7 2024-12-11T06:16:15,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/189b0b5f7dd94d6f9ba4b93f287c11fe 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/dcc7f8e3d74a41858cadd26e05fc2b6f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/dcc7f8e3d74a41858cadd26e05fc2b6f 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/811af9365a60473f8e12f60f0b72bd9f 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c83da71e8c9a4faf8fdfed8c13a94a51 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c83da71e8c9a4faf8fdfed8c13a94a51 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/8d8bc15318f54cefa078b7cab2c33a78 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/800bc8c27d3c4a2ea3a23b109b55dde5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/800bc8c27d3c4a2ea3a23b109b55dde5 2024-12-11T06:16:15,193 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/c85f808bcc534814bbcc3401d39a1eff 2024-12-11T06:16:15,194 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/04de3a495a474337bd8725b00b276477 2024-12-11T06:16:15,194 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ebfc6679684543e594cc563e01f5e4f9 2024-12-11T06:16:15,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/eba4145e716f4dc4af1ea405e674ba20 2024-12-11T06:16:15,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/ede892bfff5a463ca775d1f4e30e7401 2024-12-11T06:16:15,198 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits/438.seqid, newMaxSeqId=438, maxSeqId=4 2024-12-11T06:16:15,200 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. 2024-12-11T06:16:15,200 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 4087fec5b081d47691e33f5c7c3537f9: Waiting for close lock at 1733897772421Running coprocessor pre-close hooks at 1733897772421Disabling compacts and flushes for region at 1733897772421Disabling writes for close at 1733897773496 (+1075 ms)Obtaining lock to block concurrent updates at 1733897773496Preparing flush snapshotting stores in 4087fec5b081d47691e33f5c7c3537f9 at 1733897773496Finished memstore snapshotting TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733897773496Flushing stores of TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9. at 1733897773497 (+1 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/A: creating writer at 1733897773497Flushing mob file 4087fec5b081d47691e33f5c7c3537f9/A: appending metadata at 1733897773501 (+4 ms)Flushing mob file 4087fec5b081d47691e33f5c7c3537f9/A: closing flushed file at 1733897773501Flushing 4087fec5b081d47691e33f5c7c3537f9/A: appending metadata at 1733897773913 (+412 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/A: closing flushed file at 1733897773914 (+1 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/B: creating writer at 1733897774324 (+410 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/B: appending metadata at 1733897774326 (+2 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/B: closing flushed file at 1733897774326Flushing 4087fec5b081d47691e33f5c7c3537f9/C: creating writer at 1733897774735 (+409 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/C: appending metadata at 1733897774738 (+3 ms)Flushing 4087fec5b081d47691e33f5c7c3537f9/C: closing flushed file at 1733897774738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bfbfddc: reopening flushed file at 1733897775147 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce795a7: reopening flushed file at 1733897775151 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c6d9c7: reopening flushed file at 1733897775155 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4087fec5b081d47691e33f5c7c3537f9 in 1664ms, sequenceid=435, compaction requested=true at 1733897775161 (+6 ms)Writing region close event to WAL at 1733897775195 (+34 ms)Running coprocessor post-close hooks at 1733897775200 (+5 ms)Closed at 1733897775200 2024-12-11T06:16:15,203 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=4087fec5b081d47691e33f5c7c3537f9, regionState=CLOSED 2024-12-11T06:16:15,204 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:15,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:15,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-11T06:16:15,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 4087fec5b081d47691e33f5c7c3537f9, server=4518703b2c1d,35425,1733897658824 in 2.9440 sec 2024-12-11T06:16:15,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-11T06:16:15,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4087fec5b081d47691e33f5c7c3537f9, UNASSIGN in 2.9510 sec 2024-12-11T06:16:15,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-11T06:16:15,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9550 sec 2024-12-11T06:16:15,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897775225"}]},"ts":"1733897775225"} 2024-12-11T06:16:15,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:16:15,227 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:16:15,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9660 sec 2024-12-11T06:16:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T06:16:16,439 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:16,440 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:16:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,442 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T06:16:16,443 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,448 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,451 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits] 2024-12-11T06:16:16,455 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bbe1e8154d06448c9c8e2b729b3b2ec9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/bbe1e8154d06448c9c8e2b729b3b2ec9 2024-12-11T06:16:16,455 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8169764e243a49729f061d1afa089249 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/8169764e243a49729f061d1afa089249 2024-12-11T06:16:16,455 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/be6dea4021dc42ab9b5bc2ee4f641f8b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/A/be6dea4021dc42ab9b5bc2ee4f641f8b 2024-12-11T06:16:16,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0186a11844484a6f8be197e1df6e2755 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/0186a11844484a6f8be197e1df6e2755 2024-12-11T06:16:16,458 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb5d268936764733aac5b85dddfb6170 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/bb5d268936764733aac5b85dddfb6170 2024-12-11T06:16:16,458 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9e3e932a567343aaa3d67d0ed7c05948 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/B/9e3e932a567343aaa3d67d0ed7c05948 2024-12-11T06:16:16,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/1d453342cf0e46a1bdb8869d7cd43cd4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/1d453342cf0e46a1bdb8869d7cd43cd4 2024-12-11T06:16:16,461 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/7e3c2bffffcd44eeba09fbdde8e3b09a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/7e3c2bffffcd44eeba09fbdde8e3b09a 2024-12-11T06:16:16,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/b4fa8cebd5274597bb6b220e5a7a88bd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/C/b4fa8cebd5274597bb6b220e5a7a88bd 2024-12-11T06:16:16,464 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits/438.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9/recovered.edits/438.seqid 2024-12-11T06:16:16,465 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,465 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:16:16,466 DEBUG [PEWorker-2 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:16:16,467 DEBUG [PEWorker-2 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112b7416753b7a4b66b5aa8d94f220cc23_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112b7416753b7a4b66b5aa8d94f220cc23_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110e14253629f341e0a9b5ec3ffa6aaef4_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110e14253629f341e0a9b5ec3ffa6aaef4_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128275348a4964478b5a3ec7c662dac35_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128275348a4964478b5a3ec7c662dac35_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113885fb512a3f4944babba820f474bdec_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113885fb512a3f4944babba820f474bdec_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131e1f6ca9a1c46c1ae69ea41d322ad5a_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131e1f6ca9a1c46c1ae69ea41d322ad5a_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,475 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113e92223c9e704f8eba2eb09fee5029f4_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113e92223c9e704f8eba2eb09fee5029f4_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,476 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147627b885f844235943af50be4667814_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147627b885f844235943af50be4667814_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,476 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b6e7e485fe540e2948db9003fe908be_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b6e7e485fe540e2948db9003fe908be_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,476 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114a6321fabba54ac5ba43a4b85c673b22_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114a6321fabba54ac5ba43a4b85c673b22_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,476 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121190a43ef4e9804060a95515fe1248d5d5_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121190a43ef4e9804060a95515fe1248d5d5_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,477 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211aa8c608e8ab949c299e66e3aefa162e0_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211aa8c608e8ab949c299e66e3aefa162e0_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bee52102741844d2819014bf2e555dc2_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bee52102741844d2819014bf2e555dc2_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,477 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0f4f05992f74ae9bd9b7816e64a03f8_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d0f4f05992f74ae9bd9b7816e64a03f8_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,477 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e9a5f775d8394bbd8016edf0c10e91d1_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e9a5f775d8394bbd8016edf0c10e91d1_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,477 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e39828cfd21d45029eb51b675eeff80b_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e39828cfd21d45029eb51b675eeff80b_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,478 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f1490b658a1548e59e82c39fc9d4c2d5_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f1490b658a1548e59e82c39fc9d4c2d5_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,478 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0a28d7bea1e4ad69bd0a989ae84d124_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0a28d7bea1e4ad69bd0a989ae84d124_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,479 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e15c44b409764e4f9e8c7cdd3e50e28b_4087fec5b081d47691e33f5c7c3537f9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e15c44b409764e4f9e8c7cdd3e50e28b_4087fec5b081d47691e33f5c7c3537f9 2024-12-11T06:16:16,480 DEBUG [PEWorker-2 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:16:16,482 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,485 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:16:16,488 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:16:16,490 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,490 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:16:16,490 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897776490"}]},"ts":"9223372036854775807"} 2024-12-11T06:16:16,493 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:16:16,493 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4087fec5b081d47691e33f5c7c3537f9, NAME => 'TestAcidGuarantees,,1733897748814.4087fec5b081d47691e33f5c7c3537f9.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:16:16,493 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:16:16,493 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897776493"}]},"ts":"9223372036854775807"} 2024-12-11T06:16:16,495 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:16:16,496 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-12-11T06:16:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T06:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:16:16,553 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:16,562 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=255 (was 257), OpenFileDescriptor=457 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=891 (was 849) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=543 (was 793) 2024-12-11T06:16:16,573 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=891, ProcessCount=11, AvailableMemoryMB=533 2024-12-11T06:16:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:16:16,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:16:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:16,578 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:16:16,578 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:16,580 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-11T06:16:16,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:16:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T06:16:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742248_1424 (size=1061) 2024-12-11T06:16:16,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T06:16:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T06:16:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T06:16:17,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:16:17,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742249_1425 (size=53) 2024-12-11T06:16:17,023 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:16:17,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 520aa7289fa4bbc233d80abb04a82df6, disabling compactions & flushes 2024-12-11T06:16:17,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. after waiting 0 ms 2024-12-11T06:16:17,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 520aa7289fa4bbc233d80abb04a82df6: Waiting for close lock at 1733897777024Disabling compacts and flushes for region at 1733897777024Disabling writes for close at 1733897777024Writing region close event to WAL at 1733897777024Closed at 1733897777024 2024-12-11T06:16:17,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:16:17,026 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897777026"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897777026"}]},"ts":"1733897777026"} 2024-12-11T06:16:17,028 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:16:17,029 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:16:17,029 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897777029"}]},"ts":"1733897777029"} 2024-12-11T06:16:17,031 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:16:17,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, ASSIGN}] 2024-12-11T06:16:17,032 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, ASSIGN 2024-12-11T06:16:17,033 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:16:17,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=520aa7289fa4bbc233d80abb04a82df6, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:17,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, ASSIGN because future has completed 2024-12-11T06:16:17,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T06:16:17,344 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,344 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:16:17,345 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,345 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:16:17,345 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,345 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,347 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,349 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:17,349 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 520aa7289fa4bbc233d80abb04a82df6 columnFamilyName A 2024-12-11T06:16:17,349 DEBUG [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:17,350 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(327): Store=520aa7289fa4bbc233d80abb04a82df6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:17,350 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,353 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:17,353 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 520aa7289fa4bbc233d80abb04a82df6 columnFamilyName B 2024-12-11T06:16:17,353 DEBUG [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:17,354 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(327): Store=520aa7289fa4bbc233d80abb04a82df6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:17,355 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,356 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:17,357 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 520aa7289fa4bbc233d80abb04a82df6 columnFamilyName C 2024-12-11T06:16:17,357 DEBUG [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:17,357 INFO [StoreOpener-520aa7289fa4bbc233d80abb04a82df6-1 {}] regionserver.HStore(327): Store=520aa7289fa4bbc233d80abb04a82df6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:17,358 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,358 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,359 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,359 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,361 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,361 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,361 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:16:17,363 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,365 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:16:17,366 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 520aa7289fa4bbc233d80abb04a82df6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61907873, jitterRate=-0.07750080525875092}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:16:17,366 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,366 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 520aa7289fa4bbc233d80abb04a82df6: Running coprocessor pre-open hook at 1733897777345Writing region info on filesystem at 1733897777345Initializing all the Stores at 1733897777346 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897777346Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897777347 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897777347Cleaning up temporary data from old regions at 1733897777361 (+14 ms)Running coprocessor post-open hooks at 1733897777366 (+5 ms)Region opened successfully at 1733897777366 2024-12-11T06:16:17,367 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., pid=112, masterSystemTime=1733897777339 2024-12-11T06:16:17,370 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,370 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:17,371 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=520aa7289fa4bbc233d80abb04a82df6, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:17,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:17,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-11T06:16:17,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 in 187 msec 2024-12-11T06:16:17,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-11T06:16:17,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, ASSIGN in 346 msec 2024-12-11T06:16:17,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:16:17,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897777382"}]},"ts":"1733897777382"} 2024-12-11T06:16:17,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:16:17,389 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:16:17,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 814 msec 2024-12-11T06:16:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-11T06:16:17,760 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:17,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9e45f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,763 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6fbaee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa6b45e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,767 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,773 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@df6313e 2024-12-11T06:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:16:17,777 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:16:17,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3734923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,781 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e737dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ce3c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1db034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58fb4b74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a88cb41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,801 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d6a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2457f5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab2734a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de6e594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c37c29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1101f858, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f86346d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35caaccb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,842 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634103e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,847 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b7e2139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62811f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203a9781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c25862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,875 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da0609b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4025740, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:17,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:17,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:17,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a343af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:17,890 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b79f220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:17,896 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,896 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,897 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,901 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-11T06:16:17,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:17,911 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:17,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:17,922 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:17,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:17,929 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:17,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:17,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,937 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,938 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,955 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,961 DEBUG [Thread-1922 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,962 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:17,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:17,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2] 2024-12-11T06:16:17,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/914d2064d4d34c78b8837f190a3febb6 is 50, key is test_row_0/A:col10/1733897777927/Put/seqid=0 2024-12-11T06:16:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:18,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897778003, max timestamp=1733897778026Num uniques 859; 2024-12-11T06:16:18,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:18,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897778005, max timestamp=1733897778026Num uniques 859; 2024-12-11T06:16:18,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T06:16:18,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733897778005, max timestamp=1733897778026Num uniques 859; 2024-12-11T06:16:18,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T06:16:18,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897838095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733897778058, max timestamp=1733897778108Num uniques 1068; 2024-12-11T06:16:18,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-11T06:16:18,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733897778058, max timestamp=1733897778108Num uniques 1009; 2024-12-11T06:16:18,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-11T06:16:18,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733897778058, max timestamp=1733897778108Num uniques 1009; 2024-12-11T06:16:18,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=14558, suffix heap size=132904,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=318 2024-12-11T06:16:18,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742250_1426 (size=14341) 2024-12-11T06:16:18,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897838178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897838179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897838182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897838193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897838218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897838298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897838298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897838299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897838309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897838441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897838508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897838509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897838519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/914d2064d4d34c78b8837f190a3febb6 2024-12-11T06:16:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897838530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:18,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b0f6fb5673ca4365a92f6fcb97e59aa0 is 50, key is test_row_0/B:col10/1733897777927/Put/seqid=0 2024-12-11T06:16:18,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742251_1427 (size=12001) 2024-12-11T06:16:18,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897838760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897838829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897838829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897838838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:18,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897838849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:18,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b0f6fb5673ca4365a92f6fcb97e59aa0 2024-12-11T06:16:19,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/6ddfc748b04643c389a76478576d8cb0 is 50, key is test_row_0/C:col10/1733897777927/Put/seqid=0 2024-12-11T06:16:19,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742252_1428 (size=12001) 2024-12-11T06:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:19,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897839268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:19,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:19,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:19,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897839339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897839348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897839348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897839359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:19,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/6ddfc748b04643c389a76478576d8cb0 2024-12-11T06:16:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/914d2064d4d34c78b8837f190a3febb6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6 2024-12-11T06:16:19,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6, entries=200, sequenceid=11, filesize=14.0 K 2024-12-11T06:16:19,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b0f6fb5673ca4365a92f6fcb97e59aa0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0 2024-12-11T06:16:19,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0, entries=150, sequenceid=11, filesize=11.7 K 2024-12-11T06:16:19,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/6ddfc748b04643c389a76478576d8cb0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0 2024-12-11T06:16:19,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0, entries=150, sequenceid=11, filesize=11.7 K 2024-12-11T06:16:19,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 520aa7289fa4bbc233d80abb04a82df6 in 1518ms, sequenceid=11, compaction requested=false 2024-12-11T06:16:19,446 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-11T06:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:19,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-11T06:16:19,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:19,465 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=140.89 KB heapSize=390 KB 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:19,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/97f898f3120e41b5960b421674128797 is 50, key is test_row_0/A:col10/1733897778183/Put/seqid=0 2024-12-11T06:16:19,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742253_1429 (size=12001) 2024-12-11T06:16:19,892 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/97f898f3120e41b5960b421674128797 2024-12-11T06:16:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b956318618894a7183538ca5e35d935b is 50, key is test_row_0/B:col10/1733897778183/Put/seqid=0 2024-12-11T06:16:19,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742254_1430 (size=12001) 2024-12-11T06:16:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:20,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:20,314 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b956318618894a7183538ca5e35d935b 2024-12-11T06:16:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897840322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/74ad4532ebe24c389237dcceaa8e29e5 is 50, key is test_row_0/C:col10/1733897778183/Put/seqid=0 2024-12-11T06:16:20,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897840349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742255_1431 (size=12001) 2024-12-11T06:16:20,357 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897840358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897840358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,367 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,367 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897840381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,387 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897840439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897840648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:20,714 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:16:20,758 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/74ad4532ebe24c389237dcceaa8e29e5 2024-12-11T06:16:20,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/97f898f3120e41b5960b421674128797 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797 2024-12-11T06:16:20,764 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797, entries=150, sequenceid=63, filesize=11.7 K 2024-12-11T06:16:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-11T06:16:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b956318618894a7183538ca5e35d935b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b 2024-12-11T06:16:20,768 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b, entries=150, sequenceid=63, filesize=11.7 K 2024-12-11T06:16:20,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/74ad4532ebe24c389237dcceaa8e29e5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5 2024-12-11T06:16:20,773 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5, entries=150, sequenceid=63, filesize=11.7 K 2024-12-11T06:16:20,774 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~389.95 KB/399312, currentSize=46.96 KB/48090 for 520aa7289fa4bbc233d80abb04a82df6 in 1309ms, sequenceid=63, compaction requested=false 2024-12-11T06:16:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:20,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-11T06:16:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-11T06:16:20,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-11T06:16:20,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8640 sec 2024-12-11T06:16:20,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.8770 sec 2024-12-11T06:16:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:20,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:20,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:20,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/647855a2970c4b4bb51e76a023cd0709 is 50, key is test_row_0/A:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:20,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742256_1432 (size=14341) 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897781078, max timestamp=1733897781118Num uniques 859; 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897781078, max timestamp=1733897781118Num uniques 859; 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897781078, max timestamp=1733897781118Num uniques 859; 2024-12-11T06:16:21,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897841242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:21,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897841359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:21,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/647855a2970c4b4bb51e76a023cd0709 2024-12-11T06:16:21,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/7a6cf03912e04e679f07006ac584eb6b is 50, key is test_row_0/B:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:21,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742257_1433 (size=12001) 2024-12-11T06:16:21,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897841569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:21,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/7a6cf03912e04e679f07006ac584eb6b 2024-12-11T06:16:21,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 is 50, key is test_row_0/C:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:21,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742258_1434 (size=12001) 2024-12-11T06:16:21,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897841880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T06:16:22,079 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:22,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-11T06:16:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:22,082 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:22,083 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:22,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:22,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 2024-12-11T06:16:22,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/647855a2970c4b4bb51e76a023cd0709 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709 2024-12-11T06:16:22,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709, entries=200, sequenceid=74, filesize=14.0 K 2024-12-11T06:16:22,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/7a6cf03912e04e679f07006ac584eb6b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b 2024-12-11T06:16:22,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b, entries=150, sequenceid=74, filesize=11.7 K 2024-12-11T06:16:22,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 2024-12-11T06:16:22,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098, entries=150, sequenceid=74, filesize=11.7 K 2024-12-11T06:16:22,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 520aa7289fa4bbc233d80abb04a82df6 in 1262ms, sequenceid=74, compaction requested=true 2024-12-11T06:16:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:22,236 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:22,236 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:22,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:22,237 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:22,237 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:22,237 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,237 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=35.2 K 2024-12-11T06:16:22,237 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:22,237 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:22,237 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,237 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=39.7 K 2024-12-11T06:16:22,238 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f6fb5673ca4365a92f6fcb97e59aa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733897777918 2024-12-11T06:16:22,238 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914d2064d4d34c78b8837f190a3febb6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733897777914 2024-12-11T06:16:22,239 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b956318618894a7183538ca5e35d935b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733897778177 2024-12-11T06:16:22,241 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f898f3120e41b5960b421674128797, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733897778177 2024-12-11T06:16:22,241 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a6cf03912e04e679f07006ac584eb6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:22,243 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647855a2970c4b4bb51e76a023cd0709, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:22,250 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:22,250 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/06d422edcf8c4b8a86a54e42c4046801 is 50, key is test_row_0/A:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:22,250 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:22,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/c01e831ae3ca4176a9fc597bc2d9c454 is 50, key is test_row_0/B:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:22,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742259_1435 (size=12104) 2024-12-11T06:16:22,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742260_1436 (size=12104) 2024-12-11T06:16:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:22,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=134.18 KB heapSize=385.45 KB 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T06:16:22,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/6dca3cf023404090a8773805bec1cdba is 50, key is test_row_0/A:col10/1733897782386/Put/seqid=0 2024-12-11T06:16:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742261_1437 (size=16681) 2024-12-11T06:16:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897842397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897842400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897842403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897842407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897842408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,418 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897842518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897842518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897842518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:22,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897842529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,668 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/06d422edcf8c4b8a86a54e42c4046801 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/06d422edcf8c4b8a86a54e42c4046801 2024-12-11T06:16:22,673 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into 06d422edcf8c4b8a86a54e42c4046801(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:22,673 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:22,673 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=13, startTime=1733897782235; duration=0sec 2024-12-11T06:16:22,673 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:22,673 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:22,673 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:22,674 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:22,674 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:22,674 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,674 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=35.2 K 2024-12-11T06:16:22,674 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ddfc748b04643c389a76478576d8cb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733897777918 2024-12-11T06:16:22,675 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ad4532ebe24c389237dcceaa8e29e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733897778177 2024-12-11T06:16:22,675 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebbc9f5fc71b4c2eb98d16dbc9ee2098, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:22,678 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/c01e831ae3ca4176a9fc597bc2d9c454 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/c01e831ae3ca4176a9fc597bc2d9c454 2024-12-11T06:16:22,682 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into c01e831ae3ca4176a9fc597bc2d9c454(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:22,682 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:22,682 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=13, startTime=1733897782236; duration=0sec 2024-12-11T06:16:22,682 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:22,682 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:22,683 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:22,684 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/10d2ff65215941449665092e9825d8dc is 50, key is test_row_0/C:col10/1733897780301/Put/seqid=0 2024-12-11T06:16:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742262_1438 (size=12104) 2024-12-11T06:16:22,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:22,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897842739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897842739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897842739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897842748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:22,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/6dca3cf023404090a8773805bec1cdba 2024-12-11T06:16:22,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:22,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:22,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:22,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/d9b056102b9b45d18cb98781e5f8eb13 is 50, key is test_row_0/B:col10/1733897782386/Put/seqid=0 2024-12-11T06:16:22,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742263_1439 (size=12001) 2024-12-11T06:16:23,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:23,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:23,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:23,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:23,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:23,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/d9b056102b9b45d18cb98781e5f8eb13 2024-12-11T06:16:23,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:23,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:23,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,534 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/10d2ff65215941449665092e9825d8dc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/10d2ff65215941449665092e9825d8dc 2024-12-11T06:16:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:23,540 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into 10d2ff65215941449665092e9825d8dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:23,540 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:23,540 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=13, startTime=1733897782236; duration=0sec 2024-12-11T06:16:23,541 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:23,541 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897843530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897843530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:23,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897843531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:23,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:23,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897843532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:23,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:23,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/9a0615bbf19140d894793a631eca3936 is 50, key is test_row_0/C:col10/1733897782386/Put/seqid=0 2024-12-11T06:16:23,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742264_1440 (size=12001) 2024-12-11T06:16:23,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/9a0615bbf19140d894793a631eca3936 2024-12-11T06:16:23,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/6dca3cf023404090a8773805bec1cdba as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba 2024-12-11T06:16:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba, entries=250, sequenceid=112, filesize=16.3 K 2024-12-11T06:16:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/d9b056102b9b45d18cb98781e5f8eb13 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13 2024-12-11T06:16:23,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13, entries=150, sequenceid=112, filesize=11.7 K 2024-12-11T06:16:23,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/9a0615bbf19140d894793a631eca3936 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936 2024-12-11T06:16:23,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936, entries=150, sequenceid=112, filesize=11.7 K 2024-12-11T06:16:23,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 520aa7289fa4bbc233d80abb04a82df6 in 1194ms, sequenceid=112, compaction requested=false 2024-12-11T06:16:23,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:23,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:23,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:23,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/360b1dd284ad4255a64e55b6cea99181 is 50, key is test_row_0/A:col10/1733897782402/Put/seqid=0 2024-12-11T06:16:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742265_1441 (size=12001) 2024-12-11T06:16:24,185 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/360b1dd284ad4255a64e55b6cea99181 2024-12-11T06:16:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:24,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/216e8da20ed240009da23266e8b199c9 is 50, key is test_row_0/B:col10/1733897782402/Put/seqid=0 2024-12-11T06:16:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742266_1442 (size=12001) 2024-12-11T06:16:24,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897784208, max timestamp=1733897784226Num uniques 859; 2024-12-11T06:16:24,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897784208, max timestamp=1733897784226Num uniques 859; 2024-12-11T06:16:24,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:16:24,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:16:24,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897784208, max timestamp=1733897784226Num uniques 859; 2024-12-11T06:16:24,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:16:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897844255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897844261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897844263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897844369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897844369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897844379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897844578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897844579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897844588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,603 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/216e8da20ed240009da23266e8b199c9 2024-12-11T06:16:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/12297077364c471c9382445aea18dde6 is 50, key is test_row_0/C:col10/1733897782402/Put/seqid=0 2024-12-11T06:16:24,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742267_1443 (size=12001) 2024-12-11T06:16:24,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897844651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897844888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897844888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:24,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897844899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:25,012 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/12297077364c471c9382445aea18dde6 2024-12-11T06:16:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/360b1dd284ad4255a64e55b6cea99181 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181 2024-12-11T06:16:25,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181, entries=150, sequenceid=124, filesize=11.7 K 2024-12-11T06:16:25,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/216e8da20ed240009da23266e8b199c9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9 2024-12-11T06:16:25,022 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9, entries=150, sequenceid=124, filesize=11.7 K 2024-12-11T06:16:25,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/12297077364c471c9382445aea18dde6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6 2024-12-11T06:16:25,027 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6, entries=150, sequenceid=124, filesize=11.7 K 2024-12-11T06:16:25,028 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 520aa7289fa4bbc233d80abb04a82df6 in 1342ms, sequenceid=124, compaction requested=true 2024-12-11T06:16:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-11T06:16:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-11T06:16:25,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-11T06:16:25,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9460 sec 2024-12-11T06:16:25,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.9540 sec 2024-12-11T06:16:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:25,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897784260, max timestamp=1733897785404Num uniques 1118; 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897784260, max timestamp=1733897785404Num uniques 1118; 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733897784260, max timestamp=1733897785404Num uniques 1118; 2024-12-11T06:16:25,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:25,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/71de6642627d49069974a942df6458d1 is 50, key is test_row_0/A:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:25,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742268_1444 (size=14541) 2024-12-11T06:16:25,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/71de6642627d49069974a942df6458d1 2024-12-11T06:16:25,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/9386b359ea4d4d8e8245c502d167e7bd is 50, key is test_row_0/B:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742269_1445 (size=12151) 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785438, max timestamp=1733897785466Num uniques 859; 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785438, max timestamp=1733897785466Num uniques 859; 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785438, max timestamp=1733897785466Num uniques 859; 2024-12-11T06:16:25,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1733897785467, max timestamp=1733897785538Num uniques 1018; 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1733897785467, max timestamp=1733897785538Num uniques 1018; 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=206, cellSize=9426, totalHeapSize=21664, min timestamp=1733897785467, max timestamp=1733897785538Num uniques 1018; 2024-12-11T06:16:25,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9426, suffix heap size=133840,new segment heap size=21664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=206 2024-12-11T06:16:25,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785584, max timestamp=1733897785623Num uniques 1077; 2024-12-11T06:16:25,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-11T06:16:25,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785584, max timestamp=1733897785623Num uniques 1077; 2024-12-11T06:16:25,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-11T06:16:25,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785584, max timestamp=1733897785623Num uniques 1065; 2024-12-11T06:16:25,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48780, new segment data size=8102, suffix heap size=138728,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1065, new segment cells count=177 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897785638, max timestamp=1733897785685Num uniques 1036; 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897785638, max timestamp=1733897785685Num uniques 1036; 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733897785638, max timestamp=1733897785685Num uniques 1036; 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-11T06:16:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-11T06:16:25,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733897785720, max timestamp=1733897785734Num uniques 1145; 2024-12-11T06:16:25,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-11T06:16:25,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733897785720, max timestamp=1733897785734Num uniques 1145; 2024-12-11T06:16:25,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1733897785720, max timestamp=1733897785734Num uniques 1145; 2024-12-11T06:16:25,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-11T06:16:25,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-12-11T06:16:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897845772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897845772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:25,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897845781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:25,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897845782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:25,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/9386b359ea4d4d8e8245c502d167e7bd 2024-12-11T06:16:25,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/b722496ae7c044deb6825e0a2c370b65 is 50, key is test_row_0/C:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:25,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742270_1446 (size=12151) 2024-12-11T06:16:25,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/b722496ae7c044deb6825e0a2c370b65 2024-12-11T06:16:25,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/71de6642627d49069974a942df6458d1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1 2024-12-11T06:16:25,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1, entries=200, sequenceid=162, filesize=14.2 K 2024-12-11T06:16:25,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/9386b359ea4d4d8e8245c502d167e7bd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd 2024-12-11T06:16:25,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd, entries=150, sequenceid=162, filesize=11.9 K 2024-12-11T06:16:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/b722496ae7c044deb6825e0a2c370b65 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65 2024-12-11T06:16:25,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65, entries=150, sequenceid=162, filesize=11.9 K 2024-12-11T06:16:25,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=167.72 KB/171750 for 520aa7289fa4bbc233d80abb04a82df6 in 458ms, sequenceid=162, compaction requested=true 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:25,864 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:25,864 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:25,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55327 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:25,865 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:25,865 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:25,865 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/c01e831ae3ca4176a9fc597bc2d9c454, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=47.1 K 2024-12-11T06:16:25,865 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/06d422edcf8c4b8a86a54e42c4046801, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=54.0 K 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c01e831ae3ca4176a9fc597bc2d9c454, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d422edcf8c4b8a86a54e42c4046801, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dca3cf023404090a8773805bec1cdba, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733897781154 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b056102b9b45d18cb98781e5f8eb13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733897781154 2024-12-11T06:16:25,865 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360b1dd284ad4255a64e55b6cea99181, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733897782399 2024-12-11T06:16:25,866 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 216e8da20ed240009da23266e8b199c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733897782399 2024-12-11T06:16:25,866 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71de6642627d49069974a942df6458d1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:25,866 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9386b359ea4d4d8e8245c502d167e7bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:25,874 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:25,875 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/ce67a5ec22954197bc310dfbf97ce72c is 50, key is test_row_0/B:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:25,875 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:25,876 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/c9b5a6eea0504099ad7f551d98d5c79b is 50, key is test_row_0/A:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:25,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742271_1447 (size=12391) 2024-12-11T06:16:25,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742272_1448 (size=12391) 2024-12-11T06:16:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:25,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=174.43 KB heapSize=483.09 KB 2024-12-11T06:16:25,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897785770, max timestamp=1733897785898Num uniques 1254; 2024-12-11T06:16:25,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897785770, max timestamp=1733897785898Num uniques 1254; 2024-12-11T06:16:25,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-11T06:16:25,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-11T06:16:25,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733897785770, max timestamp=1733897785898Num uniques 1254; 2024-12-11T06:16:25,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:25,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:25,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/e979f713c6de44a184f990771e14786e is 50, key is test_row_0/A:col10/1733897785770/Put/seqid=0 2024-12-11T06:16:25,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742273_1449 (size=17181) 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785933, max timestamp=1733897785944Num uniques 859; 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785933, max timestamp=1733897785944Num uniques 859; 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897785933, max timestamp=1733897785944Num uniques 859; 2024-12-11T06:16:25,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:25,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897785972, max timestamp=1733897785981Num uniques 1018; 2024-12-11T06:16:25,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:16:25,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897785972, max timestamp=1733897785981Num uniques 1018; 2024-12-11T06:16:25,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:16:25,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733897785972, max timestamp=1733897785981Num uniques 1018; 2024-12-11T06:16:25,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T06:16:26,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785998, max timestamp=1733897786057Num uniques 1077; 2024-12-11T06:16:26,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-11T06:16:26,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785998, max timestamp=1733897786057Num uniques 1077; 2024-12-11T06:16:26,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-11T06:16:26,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733897785998, max timestamp=1733897786057Num uniques 1077; 2024-12-11T06:16:26,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-11T06:16:26,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733897786088, max timestamp=1733897786108Num uniques 1036; 2024-12-11T06:16:26,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-11T06:16:26,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733897786088, max timestamp=1733897786108Num uniques 1036; 2024-12-11T06:16:26,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-11T06:16:26,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733897786088, max timestamp=1733897786108Num uniques 1036; 2024-12-11T06:16:26,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-12-11T06:16:26,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897846152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897846153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897846156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897846158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897846268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897846268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897846268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897846268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,291 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/ce67a5ec22954197bc310dfbf97ce72c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ce67a5ec22954197bc310dfbf97ce72c 2024-12-11T06:16:26,291 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/c9b5a6eea0504099ad7f551d98d5c79b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c9b5a6eea0504099ad7f551d98d5c79b 2024-12-11T06:16:26,295 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into c9b5a6eea0504099ad7f551d98d5c79b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:26,295 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into ce67a5ec22954197bc310dfbf97ce72c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:26,295 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=12, startTime=1733897785864; duration=0sec 2024-12-11T06:16:26,295 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=12, startTime=1733897785864; duration=0sec 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:26,295 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:26,296 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:26,296 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:26,296 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:26,296 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/10d2ff65215941449665092e9825d8dc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=47.1 K 2024-12-11T06:16:26,296 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d2ff65215941449665092e9825d8dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733897780301 2024-12-11T06:16:26,296 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0615bbf19140d894793a631eca3936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733897781154 2024-12-11T06:16:26,296 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 12297077364c471c9382445aea18dde6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733897782399 2024-12-11T06:16:26,297 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b722496ae7c044deb6825e0a2c370b65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:26,304 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:26,304 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/117024a8e3974a4d9b7bd3c014901143 is 50, key is test_row_0/C:col10/1733897785404/Put/seqid=0 2024-12-11T06:16:26,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742274_1450 (size=12391) 2024-12-11T06:16:26,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/e979f713c6de44a184f990771e14786e 2024-12-11T06:16:26,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/dba27c01f2394cab8bf50d7b7a3aa20a is 50, key is test_row_0/B:col10/1733897785770/Put/seqid=0 2024-12-11T06:16:26,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742275_1451 (size=12301) 2024-12-11T06:16:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897846468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,475 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897846478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897846478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897846478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897846478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-11T06:16:26,569 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:26,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-11T06:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:26,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:26,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:26,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:26,710 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/117024a8e3974a4d9b7bd3c014901143 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/117024a8e3974a4d9b7bd3c014901143 2024-12-11T06:16:26,713 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into 117024a8e3974a4d9b7bd3c014901143(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:26,713 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:26,713 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=12, startTime=1733897785864; duration=0sec 2024-12-11T06:16:26,713 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:26,713 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:26,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/dba27c01f2394cab8bf50d7b7a3aa20a 2024-12-11T06:16:26,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 is 50, key is test_row_0/C:col10/1733897785770/Put/seqid=0 2024-12-11T06:16:26,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-11T06:16:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:26,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742276_1452 (size=12301) 2024-12-11T06:16:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897846798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897846799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897846799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897846799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:26,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-11T06:16:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:26,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:27,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-11T06:16:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:27,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:27,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 2024-12-11T06:16:27,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/e979f713c6de44a184f990771e14786e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e 2024-12-11T06:16:27,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e, entries=250, sequenceid=270, filesize=16.8 K 2024-12-11T06:16:27,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/dba27c01f2394cab8bf50d7b7a3aa20a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a 2024-12-11T06:16:27,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a, entries=150, sequenceid=270, filesize=12.0 K 2024-12-11T06:16:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 2024-12-11T06:16:27,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3, entries=150, sequenceid=270, filesize=12.0 K 2024-12-11T06:16:27,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=154.31 KB/158010 for 520aa7289fa4bbc233d80abb04a82df6 in 1246ms, sequenceid=270, compaction requested=false 2024-12-11T06:16:27,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:27,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-11T06:16:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:27,181 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=154.31 KB heapSize=428.16 KB 2024-12-11T06:16:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/c17c6b540ab24d91b30f2dcfe09c90b4 is 50, key is test_row_0/A:col10/1733897786157/Put/seqid=0 2024-12-11T06:16:27,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742277_1453 (size=12301) 2024-12-11T06:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897847328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897847329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897847329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897847337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897847449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897847449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897847449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897847449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,590 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/c17c6b540ab24d91b30f2dcfe09c90b4 2024-12-11T06:16:27,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/311c886a757346e3b814c2d1d440a7f1 is 50, key is test_row_0/B:col10/1733897786157/Put/seqid=0 2024-12-11T06:16:27,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742278_1454 (size=12301) 2024-12-11T06:16:27,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897847659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897847659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897847659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897847659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897847968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897847969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897847969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897847969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,007 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/311c886a757346e3b814c2d1d440a7f1 2024-12-11T06:16:28,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/a93019a6434d44b38169cbd485938d3c is 50, key is test_row_0/C:col10/1733897786157/Put/seqid=0 2024-12-11T06:16:28,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742279_1455 (size=12301) 2024-12-11T06:16:28,416 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/a93019a6434d44b38169cbd485938d3c 2024-12-11T06:16:28,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/c17c6b540ab24d91b30f2dcfe09c90b4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4 2024-12-11T06:16:28,423 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4, entries=150, sequenceid=361, filesize=12.0 K 2024-12-11T06:16:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/311c886a757346e3b814c2d1d440a7f1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1 2024-12-11T06:16:28,427 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1, entries=150, sequenceid=361, filesize=12.0 K 2024-12-11T06:16:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/a93019a6434d44b38169cbd485938d3c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c 2024-12-11T06:16:28,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c, entries=150, sequenceid=361, filesize=12.0 K 2024-12-11T06:16:28,433 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.11 KB/438384, currentSize=33.54 KB/34350 for 520aa7289fa4bbc233d80abb04a82df6 in 1251ms, sequenceid=361, compaction requested=true 2024-12-11T06:16:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:28,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-11T06:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-11T06:16:28,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-11T06:16:28,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-12-11T06:16:28,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8780 sec 2024-12-11T06:16:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:28,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:28,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:28,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/65009073a27142d99973a553056cb4a9 is 50, key is test_row_0/A:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742280_1456 (size=14741) 2024-12-11T06:16:28,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733897788494, max timestamp=1733897788521Num uniques 859; 2024-12-11T06:16:28,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T06:16:28,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733897788494, max timestamp=1733897788521Num uniques 859; 2024-12-11T06:16:28,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T06:16:28,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733897788494, max timestamp=1733897788521Num uniques 859; 2024-12-11T06:16:28,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T06:16:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897848547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897848552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897848556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897848557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897848668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897848668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897848668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897848668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T06:16:28,719 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:28,719 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-11T06:16:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:28,721 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:28,722 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:28,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:28,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:28,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:28,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897848888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897848888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897848888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897848888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:28,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:28,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:28,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/65009073a27142d99973a553056cb4a9 2024-12-11T06:16:28,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5968bea3a92346ffa7e269fc7c44436a is 50, key is test_row_0/B:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:28,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742281_1457 (size=12301) 2024-12-11T06:16:29,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:29,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897849199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897849199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897849199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897849199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5968bea3a92346ffa7e269fc7c44436a 2024-12-11T06:16:29,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ade4283c235d4603bdbb670e9da90b64 is 50, key is test_row_0/C:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:29,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742282_1458 (size=12301) 2024-12-11T06:16:29,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:29,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:29,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:29,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ade4283c235d4603bdbb670e9da90b64 2024-12-11T06:16:29,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/65009073a27142d99973a553056cb4a9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9 2024-12-11T06:16:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897849709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9, entries=200, sequenceid=372, filesize=14.4 K 2024-12-11T06:16:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897849709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897849709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897849709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:29,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5968bea3a92346ffa7e269fc7c44436a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a 2024-12-11T06:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:29,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a, entries=150, sequenceid=372, filesize=12.0 K 2024-12-11T06:16:29,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ade4283c235d4603bdbb670e9da90b64 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64 2024-12-11T06:16:29,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64, entries=150, sequenceid=372, filesize=12.0 K 2024-12-11T06:16:29,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 520aa7289fa4bbc233d80abb04a82df6 in 1240ms, sequenceid=372, compaction requested=true 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:29,722 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:29,722 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:29,723 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:29,723 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:29,723 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,723 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ce67a5ec22954197bc310dfbf97ce72c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=48.1 K 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ce67a5ec22954197bc310dfbf97ce72c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:29,724 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,724 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c9b5a6eea0504099ad7f551d98d5c79b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=55.3 K 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting dba27c01f2394cab8bf50d7b7a3aa20a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733897785770 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b5a6eea0504099ad7f551d98d5c79b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 311c886a757346e3b814c2d1d440a7f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733897786136 2024-12-11T06:16:29,724 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e979f713c6de44a184f990771e14786e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733897785770 2024-12-11T06:16:29,725 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 5968bea3a92346ffa7e269fc7c44436a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:29,725 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c17c6b540ab24d91b30f2dcfe09c90b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733897786136 2024-12-11T06:16:29,725 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65009073a27142d99973a553056cb4a9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:29,731 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:29,731 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/a3d39e8ca4b1499c8d66e64cf7798bde is 50, key is test_row_0/A:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:29,732 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:29,732 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/6ae0a65da7e14a58a2073905400b6212 is 50, key is test_row_0/B:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:29,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742283_1459 (size=12677) 2024-12-11T06:16:29,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742284_1460 (size=12677) 2024-12-11T06:16:29,738 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/a3d39e8ca4b1499c8d66e64cf7798bde as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a3d39e8ca4b1499c8d66e64cf7798bde 2024-12-11T06:16:29,741 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into a3d39e8ca4b1499c8d66e64cf7798bde(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:29,742 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:29,742 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=12, startTime=1733897789722; duration=0sec 2024-12-11T06:16:29,742 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:29,742 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:29,742 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:29,745 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:29,745 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:29,745 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,745 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/117024a8e3974a4d9b7bd3c014901143, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=48.1 K 2024-12-11T06:16:29,745 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 117024a8e3974a4d9b7bd3c014901143, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733897784260 2024-12-11T06:16:29,745 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbafab28b5ea45e3b5adfb5e5b51e7f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733897785770 2024-12-11T06:16:29,746 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93019a6434d44b38169cbd485938d3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733897786136 2024-12-11T06:16:29,746 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade4283c235d4603bdbb670e9da90b64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:29,751 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:29,751 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/c86b112236314f139eef5b626ca32cf2 is 50, key is test_row_0/C:col10/1733897788481/Put/seqid=0 2024-12-11T06:16:29,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742285_1461 (size=12677) 2024-12-11T06:16:29,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-11T06:16:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:29,788 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:29,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/5c4107f7f4de44bd977b255f18a45963 is 50, key is test_row_0/A:col10/1733897788554/Put/seqid=0 2024-12-11T06:16:29,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742286_1462 (size=12301) 2024-12-11T06:16:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:30,139 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/6ae0a65da7e14a58a2073905400b6212 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/6ae0a65da7e14a58a2073905400b6212 2024-12-11T06:16:30,142 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into 6ae0a65da7e14a58a2073905400b6212(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:30,142 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:30,142 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=12, startTime=1733897789722; duration=0sec 2024-12-11T06:16:30,142 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:30,142 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:30,157 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/c86b112236314f139eef5b626ca32cf2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/c86b112236314f139eef5b626ca32cf2 2024-12-11T06:16:30,160 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into c86b112236314f139eef5b626ca32cf2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:30,161 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:30,161 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=12, startTime=1733897789722; duration=0sec 2024-12-11T06:16:30,161 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:30,161 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:30,196 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/5c4107f7f4de44bd977b255f18a45963 2024-12-11T06:16:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/8cd78ac0b648413984323faabf8bf38e is 50, key is test_row_0/B:col10/1733897788554/Put/seqid=0 2024-12-11T06:16:30,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742287_1463 (size=12301) 2024-12-11T06:16:30,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/8cd78ac0b648413984323faabf8bf38e 2024-12-11T06:16:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/355bc65ea43547c986776741d2a13cce is 50, key is test_row_0/C:col10/1733897788554/Put/seqid=0 2024-12-11T06:16:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742288_1464 (size=12301) 2024-12-11T06:16:30,618 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/355bc65ea43547c986776741d2a13cce 2024-12-11T06:16:30,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/5c4107f7f4de44bd977b255f18a45963 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963 2024-12-11T06:16:30,625 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963, entries=150, sequenceid=407, filesize=12.0 K 2024-12-11T06:16:30,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/8cd78ac0b648413984323faabf8bf38e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e 2024-12-11T06:16:30,628 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e, entries=150, sequenceid=407, filesize=12.0 K 2024-12-11T06:16:30,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/355bc65ea43547c986776741d2a13cce as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce 2024-12-11T06:16:30,632 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce, entries=150, sequenceid=407, filesize=12.0 K 2024-12-11T06:16:30,632 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=0 B/0 for 520aa7289fa4bbc233d80abb04a82df6 in 845ms, sequenceid=407, compaction requested=false 2024-12-11T06:16:30,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:30,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:30,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-11T06:16:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-11T06:16:30,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-11T06:16:30,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-12-11T06:16:30,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9150 sec 2024-12-11T06:16:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:30,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:30,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:30,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/24543bad04ec4c1d9e3f60a80ccbc697 is 50, key is test_row_0/A:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:30,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742289_1465 (size=14741) 2024-12-11T06:16:30,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/24543bad04ec4c1d9e3f60a80ccbc697 2024-12-11T06:16:30,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/ec5501b73f2043b1a75cba083d549173 is 50, key is test_row_0/B:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:30,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742290_1466 (size=12301) 2024-12-11T06:16:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897850768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897850770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897790766, max timestamp=1733897790776Num uniques 859; 2024-12-11T06:16:30,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897790766, max timestamp=1733897790776Num uniques 859; 2024-12-11T06:16:30,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:30,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733897790766, max timestamp=1733897790776Num uniques 859; 2024-12-11T06:16:30,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T06:16:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897850823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897850830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-11T06:16:30,869 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:30,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-11T06:16:30,870 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:30,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:30,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897850889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897850889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897850938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897850938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:31,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T06:16:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897851099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897851099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/ec5501b73f2043b1a75cba083d549173 2024-12-11T06:16:31,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897851148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897851148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/eccc3bbc572f492ab9cf93ab285dd80e is 50, key is test_row_0/C:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:31,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742291_1467 (size=12301) 2024-12-11T06:16:31,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/eccc3bbc572f492ab9cf93ab285dd80e 2024-12-11T06:16:31,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/24543bad04ec4c1d9e3f60a80ccbc697 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697 2024-12-11T06:16:31,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697, entries=200, sequenceid=420, filesize=14.4 K 2024-12-11T06:16:31,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/ec5501b73f2043b1a75cba083d549173 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173 2024-12-11T06:16:31,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173, entries=150, sequenceid=420, filesize=12.0 K 2024-12-11T06:16:31,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/eccc3bbc572f492ab9cf93ab285dd80e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e 2024-12-11T06:16:31,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e, entries=150, sequenceid=420, filesize=12.0 K 2024-12-11T06:16:31,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 520aa7289fa4bbc233d80abb04a82df6 in 441ms, sequenceid=420, compaction requested=true 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:31,173 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T06:16:31,174 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:31,174 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:31,174 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:31,174 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:31,174 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:31,174 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,174 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,175 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a3d39e8ca4b1499c8d66e64cf7798bde, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=38.8 K 2024-12-11T06:16:31,175 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/6ae0a65da7e14a58a2073905400b6212, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.4 K 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae0a65da7e14a58a2073905400b6212, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d39e8ca4b1499c8d66e64cf7798bde, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:31,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,179 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd78ac0b648413984323faabf8bf38e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733897788534 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4107f7f4de44bd977b255f18a45963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733897788534 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5501b73f2043b1a75cba083d549173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790730 2024-12-11T06:16:31,179 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24543bad04ec4c1d9e3f60a80ccbc697, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790727 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:31,197 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:31,197 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/9119b4212e6e4704a8c8543e36e177b0 is 50, key is test_row_0/B:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/4bb4ba0fbdca4807ac8fdca2234f8e78 is 50, key is test_row_0/A:col10/1733897790829/Put/seqid=0 2024-12-11T06:16:31,198 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:31,198 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/48005924c2de412f96876968f5dd8be9 is 50, key is test_row_0/A:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742293_1469 (size=12301) 2024-12-11T06:16:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742292_1468 (size=12779) 2024-12-11T06:16:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742294_1470 (size=12779) 2024-12-11T06:16:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:31,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897851429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897851435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897851459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897851459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897851549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897851549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,604 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/4bb4ba0fbdca4807ac8fdca2234f8e78 2024-12-11T06:16:31,608 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/9119b4212e6e4704a8c8543e36e177b0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9119b4212e6e4704a8c8543e36e177b0 2024-12-11T06:16:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/97797d75ae30459f919156aa1c5ced0c is 50, key is test_row_0/B:col10/1733897790829/Put/seqid=0 2024-12-11T06:16:31,612 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into 9119b4212e6e4704a8c8543e36e177b0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:31,612 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:31,612 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=13, startTime=1733897791173; duration=0sec 2024-12-11T06:16:31,612 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:31,612 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:31,612 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:31,614 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/48005924c2de412f96876968f5dd8be9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/48005924c2de412f96876968f5dd8be9 2024-12-11T06:16:31,614 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:31,614 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:31,614 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:31,614 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/c86b112236314f139eef5b626ca32cf2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.4 K 2024-12-11T06:16:31,614 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c86b112236314f139eef5b626ca32cf2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733897787326 2024-12-11T06:16:31,614 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 355bc65ea43547c986776741d2a13cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733897788534 2024-12-11T06:16:31,615 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting eccc3bbc572f492ab9cf93ab285dd80e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790730 2024-12-11T06:16:31,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742295_1471 (size=12301) 2024-12-11T06:16:31,627 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into 48005924c2de412f96876968f5dd8be9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:31,627 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:31,627 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=13, startTime=1733897791173; duration=0sec 2024-12-11T06:16:31,627 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:31,627 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:31,631 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:31,631 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/112960f694ef47e2a1784ee0549ef64e is 50, key is test_row_0/C:col10/1733897790732/Put/seqid=0 2024-12-11T06:16:31,632 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/97797d75ae30459f919156aa1c5ced0c 2024-12-11T06:16:31,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742296_1472 (size=12779) 2024-12-11T06:16:31,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/22ee1d017b084d4fb29b03ee549952dc is 50, key is test_row_0/C:col10/1733897790829/Put/seqid=0 2024-12-11T06:16:31,643 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/112960f694ef47e2a1784ee0549ef64e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/112960f694ef47e2a1784ee0549ef64e 2024-12-11T06:16:31,648 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into 112960f694ef47e2a1784ee0549ef64e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:31,648 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:31,648 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=13, startTime=1733897791173; duration=0sec 2024-12-11T06:16:31,648 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:31,648 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:31,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742297_1473 (size=12301) 2024-12-11T06:16:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897851758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897851759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897851978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897851978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:32,051 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/22ee1d017b084d4fb29b03ee549952dc 2024-12-11T06:16:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/4bb4ba0fbdca4807ac8fdca2234f8e78 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78 2024-12-11T06:16:32,058 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78, entries=150, sequenceid=455, filesize=12.0 K 2024-12-11T06:16:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/97797d75ae30459f919156aa1c5ced0c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c 2024-12-11T06:16:32,061 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c, entries=150, sequenceid=455, filesize=12.0 K 2024-12-11T06:16:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/22ee1d017b084d4fb29b03ee549952dc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc 2024-12-11T06:16:32,065 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc, entries=150, sequenceid=455, filesize=12.0 K 2024-12-11T06:16:32,066 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 520aa7289fa4bbc233d80abb04a82df6 in 887ms, sequenceid=455, compaction requested=false 2024-12-11T06:16:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-11T06:16:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-11T06:16:32,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-11T06:16:32,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-12-11T06:16:32,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2000 sec 2024-12-11T06:16:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:32,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:32,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0739e1f4bed947a9989b9b300c3cd209 is 50, key is test_row_0/A:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:32,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742298_1474 (size=14741) 2024-12-11T06:16:32,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897852115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897852116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897852218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897852229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897852438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897852448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0739e1f4bed947a9989b9b300c3cd209 2024-12-11T06:16:32,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/a201632ac1a946a1ac9f62bfe9028029 is 50, key is test_row_0/B:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742299_1475 (size=12301) 2024-12-11T06:16:32,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897852748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897852759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:32,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/a201632ac1a946a1ac9f62bfe9028029 2024-12-11T06:16:32,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ee09cb21a476458680ddc9ae1df4f481 is 50, key is test_row_0/C:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:32,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742300_1476 (size=12301) 2024-12-11T06:16:33,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897852998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897852998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:33,003 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,003 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T06:16:33,019 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:33,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-11T06:16:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:33,020 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:33,021 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:33,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:33,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T06:16:33,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:33,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:33,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:33,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897853258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897853268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:33,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ee09cb21a476458680ddc9ae1df4f481 2024-12-11T06:16:33,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0739e1f4bed947a9989b9b300c3cd209 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209 2024-12-11T06:16:33,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209, entries=200, sequenceid=471, filesize=14.4 K 2024-12-11T06:16:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/a201632ac1a946a1ac9f62bfe9028029 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029 2024-12-11T06:16:33,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029, entries=150, sequenceid=471, filesize=12.0 K 2024-12-11T06:16:33,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/ee09cb21a476458680ddc9ae1df4f481 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481 2024-12-11T06:16:33,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481, entries=150, sequenceid=471, filesize=12.0 K 2024-12-11T06:16:33,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 520aa7289fa4bbc233d80abb04a82df6 in 1245ms, sequenceid=471, compaction requested=true 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:33,316 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:33,316 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:33,317 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,317 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,317 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9119b4212e6e4704a8c8543e36e177b0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.5 K 2024-12-11T06:16:33,317 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/48005924c2de412f96876968f5dd8be9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=38.9 K 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9119b4212e6e4704a8c8543e36e177b0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790730 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48005924c2de412f96876968f5dd8be9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790730 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 97797d75ae30459f919156aa1c5ced0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733897790808 2024-12-11T06:16:33,317 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb4ba0fbdca4807ac8fdca2234f8e78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733897790808 2024-12-11T06:16:33,318 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting a201632ac1a946a1ac9f62bfe9028029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:33,318 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0739e1f4bed947a9989b9b300c3cd209, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T06:16:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,324 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:16:33,324 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:33,325 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/3490b8b995834e509167ac6f981d367b is 50, key is test_row_0/B:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:33,329 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:33,329 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/b7c4b3bbdccf479283446a4b67eccbcb is 50, key is test_row_0/A:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0eea2f6059d04bee9d435e09a27aa1a8 is 50, key is test_row_0/A:col10/1733897792104/Put/seqid=0 2024-12-11T06:16:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742301_1477 (size=12881) 2024-12-11T06:16:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:33,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742302_1478 (size=12881) 2024-12-11T06:16:33,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742303_1479 (size=12301) 2024-12-11T06:16:33,346 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/b7c4b3bbdccf479283446a4b67eccbcb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/b7c4b3bbdccf479283446a4b67eccbcb 2024-12-11T06:16:33,350 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into b7c4b3bbdccf479283446a4b67eccbcb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:33,350 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:33,350 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=13, startTime=1733897793316; duration=0sec 2024-12-11T06:16:33,350 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:33,350 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:33,350 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:33,351 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:33,352 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:33,352 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:33,352 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/112960f694ef47e2a1784ee0549ef64e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.5 K 2024-12-11T06:16:33,352 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 112960f694ef47e2a1784ee0549ef64e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733897790730 2024-12-11T06:16:33,352 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ee1d017b084d4fb29b03ee549952dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1733897790808 2024-12-11T06:16:33,353 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee09cb21a476458680ddc9ae1df4f481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:33,359 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:33,359 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/458565b1bcde4e50b5c542500a923867 is 50, key is test_row_0/C:col10/1733897792070/Put/seqid=0 2024-12-11T06:16:33,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742304_1480 (size=12881) 2024-12-11T06:16:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:33,741 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/3490b8b995834e509167ac6f981d367b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/3490b8b995834e509167ac6f981d367b 2024-12-11T06:16:33,745 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into 3490b8b995834e509167ac6f981d367b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:33,745 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:33,745 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=13, startTime=1733897793316; duration=0sec 2024-12-11T06:16:33,745 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:33,745 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:33,745 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0eea2f6059d04bee9d435e09a27aa1a8 2024-12-11T06:16:33,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/566ce1bd618c48b098c001ef938f6ba6 is 50, key is test_row_0/B:col10/1733897792104/Put/seqid=0 2024-12-11T06:16:33,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742305_1481 (size=12301) 2024-12-11T06:16:33,767 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/458565b1bcde4e50b5c542500a923867 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/458565b1bcde4e50b5c542500a923867 2024-12-11T06:16:33,771 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into 458565b1bcde4e50b5c542500a923867(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:33,771 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:33,771 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=13, startTime=1733897793316; duration=0sec 2024-12-11T06:16:33,771 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:33,771 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:34,156 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/566ce1bd618c48b098c001ef938f6ba6 2024-12-11T06:16:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/d4fed49a692c45d8b90d3ac7874da767 is 50, key is test_row_0/C:col10/1733897792104/Put/seqid=0 2024-12-11T06:16:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742306_1482 (size=12301) 2024-12-11T06:16:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:34,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897854303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897854310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897854418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897854418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,567 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/d4fed49a692c45d8b90d3ac7874da767 2024-12-11T06:16:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/0eea2f6059d04bee9d435e09a27aa1a8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8 2024-12-11T06:16:34,575 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8, entries=150, sequenceid=490, filesize=12.0 K 2024-12-11T06:16:34,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/566ce1bd618c48b098c001ef938f6ba6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6 2024-12-11T06:16:34,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6, entries=150, sequenceid=490, filesize=12.0 K 2024-12-11T06:16:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/d4fed49a692c45d8b90d3ac7874da767 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767 2024-12-11T06:16:34,581 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767, entries=150, sequenceid=490, filesize=12.0 K 2024-12-11T06:16:34,582 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 520aa7289fa4bbc233d80abb04a82df6 in 1258ms, sequenceid=490, compaction requested=false 2024-12-11T06:16:34,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:34,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:34,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-11T06:16:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-11T06:16:34,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-11T06:16:34,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5620 sec 2024-12-11T06:16:34,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5660 sec 2024-12-11T06:16:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:34,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:34,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/837a2039eb804e7eb241e0b21ff1f5e2 is 50, key is test_row_0/A:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:34,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742307_1483 (size=17181) 2024-12-11T06:16:34,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/837a2039eb804e7eb241e0b21ff1f5e2 2024-12-11T06:16:34,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/2a0df90acf8c421f98068121e2e4fe2e is 50, key is test_row_0/B:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:34,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742308_1484 (size=12301) 2024-12-11T06:16:34,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897854686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897854688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897854798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897854808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44340 deadline: 1733897855009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897855009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,015 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44298 deadline: 1733897855018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897855018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,025 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 520aa7289fa4bbc233d80abb04a82df6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/2a0df90acf8c421f98068121e2e4fe2e 2024-12-11T06:16:35,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/63eec9ca7e174c8fa03c7032e61c475b is 50, key is test_row_0/C:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742309_1485 (size=12301) 2024-12-11T06:16:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T06:16:35,169 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:35,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-11T06:16:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:35,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:35,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:35,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897855318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T06:16:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897855329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T06:16:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:35,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/63eec9ca7e174c8fa03c7032e61c475b 2024-12-11T06:16:35,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/837a2039eb804e7eb241e0b21ff1f5e2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2 2024-12-11T06:16:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:35,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2, entries=250, sequenceid=508, filesize=16.8 K 2024-12-11T06:16:35,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/2a0df90acf8c421f98068121e2e4fe2e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e 2024-12-11T06:16:35,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e, entries=150, sequenceid=508, filesize=12.0 K 2024-12-11T06:16:35,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/63eec9ca7e174c8fa03c7032e61c475b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b 2024-12-11T06:16:35,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b, entries=150, sequenceid=508, filesize=12.0 K 2024-12-11T06:16:35,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 520aa7289fa4bbc233d80abb04a82df6 in 861ms, sequenceid=508, compaction requested=true 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:35,495 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:35,495 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:35,496 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:35,496 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:35,496 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:35,496 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:35,496 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,496 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,496 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/b7c4b3bbdccf479283446a4b67eccbcb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=41.4 K 2024-12-11T06:16:35,496 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/3490b8b995834e509167ac6f981d367b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.6 K 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c4b3bbdccf479283446a4b67eccbcb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3490b8b995834e509167ac6f981d367b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 566ce1bd618c48b098c001ef938f6ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733897792104 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eea2f6059d04bee9d435e09a27aa1a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733897792104 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837a2039eb804e7eb241e0b21ff1f5e2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:35,497 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0df90acf8c421f98068121e2e4fe2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:35,502 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:35,503 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b8810c0847044ba7bcac42e6fd24921b is 50, key is test_row_0/B:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:35,505 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#415 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:35,505 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/30d21bd5f4cc43abb757d57e98d43239 is 50, key is test_row_0/A:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:35,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742310_1486 (size=12983) 2024-12-11T06:16:35,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742311_1487 (size=12983) 2024-12-11T06:16:35,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,628 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:35,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/37070520a99147f6a86a1c62417a2cf8 is 50, key is test_row_0/A:col10/1733897794673/Put/seqid=0 2024-12-11T06:16:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742312_1488 (size=12301) 2024-12-11T06:16:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:35,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:35,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897855873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897855880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:35,918 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/b8810c0847044ba7bcac42e6fd24921b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b8810c0847044ba7bcac42e6fd24921b 2024-12-11T06:16:35,920 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into b8810c0847044ba7bcac42e6fd24921b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:35,920 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:35,921 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=13, startTime=1733897795495; duration=0sec 2024-12-11T06:16:35,921 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:35,921 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:35,921 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:35,921 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:35,921 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:35,921 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:35,922 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/458565b1bcde4e50b5c542500a923867, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.6 K 2024-12-11T06:16:35,922 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 458565b1bcde4e50b5c542500a923867, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1733897791422 2024-12-11T06:16:35,922 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d4fed49a692c45d8b90d3ac7874da767, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733897792104 2024-12-11T06:16:35,922 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 63eec9ca7e174c8fa03c7032e61c475b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:35,925 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/30d21bd5f4cc43abb757d57e98d43239 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/30d21bd5f4cc43abb757d57e98d43239 2024-12-11T06:16:35,927 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:35,928 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/101bdee5ec4a4e5a8370fe1ba64cda4f is 50, key is test_row_0/C:col10/1733897794633/Put/seqid=0 2024-12-11T06:16:35,928 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into 30d21bd5f4cc43abb757d57e98d43239(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:35,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:35,928 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=13, startTime=1733897795495; duration=0sec 2024-12-11T06:16:35,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:35,928 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:35,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742313_1489 (size=12983) 2024-12-11T06:16:35,934 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/101bdee5ec4a4e5a8370fe1ba64cda4f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/101bdee5ec4a4e5a8370fe1ba64cda4f 2024-12-11T06:16:35,937 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into 101bdee5ec4a4e5a8370fe1ba64cda4f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:35,937 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:35,937 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=13, startTime=1733897795495; duration=0sec 2024-12-11T06:16:35,937 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:35,937 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897855989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897855999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/37070520a99147f6a86a1c62417a2cf8 2024-12-11T06:16:36,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/de1cc744214946fab5d9f36365ad959b is 50, key is test_row_0/B:col10/1733897794673/Put/seqid=0 2024-12-11T06:16:36,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742314_1490 (size=12301) 2024-12-11T06:16:36,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897856198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897856208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:36,449 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/de1cc744214946fab5d9f36365ad959b 2024-12-11T06:16:36,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/2acd225caf3a479497a3d3e34b837744 is 50, key is test_row_0/C:col10/1733897794673/Put/seqid=0 2024-12-11T06:16:36,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742315_1491 (size=12301) 2024-12-11T06:16:36,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/2acd225caf3a479497a3d3e34b837744 2024-12-11T06:16:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/37070520a99147f6a86a1c62417a2cf8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8 2024-12-11T06:16:36,495 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:16:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/de1cc744214946fab5d9f36365ad959b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b 2024-12-11T06:16:36,500 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:16:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/2acd225caf3a479497a3d3e34b837744 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744 2024-12-11T06:16:36,509 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744, entries=150, sequenceid=525, filesize=12.0 K 2024-12-11T06:16:36,510 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 520aa7289fa4bbc233d80abb04a82df6 in 882ms, sequenceid=525, compaction requested=false 2024-12-11T06:16:36,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:36,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:36,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-11T06:16:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-11T06:16:36,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:36,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-11T06:16:36,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3410 sec 2024-12-11T06:16:36,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3500 sec 2024-12-11T06:16:36,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/a6403590312a4ef98e4b9f52cadbe49b is 50, key is test_row_0/A:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742316_1492 (size=14741) 2024-12-11T06:16:36,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897856581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897856585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897856586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897856698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897856709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897856710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897856918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897856919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897856919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:36,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/a6403590312a4ef98e4b9f52cadbe49b 2024-12-11T06:16:36,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/464a8bd2f2f448f4b0bd3e3c0c0e682c is 50, key is test_row_0/B:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:37,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742317_1493 (size=12301) 2024-12-11T06:16:37,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/464a8bd2f2f448f4b0bd3e3c0c0e682c 2024-12-11T06:16:37,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/271810f76f1343e4968894850f83a559 is 50, key is test_row_0/C:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:37,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742318_1494 (size=12301) 2024-12-11T06:16:37,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/271810f76f1343e4968894850f83a559 2024-12-11T06:16:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/a6403590312a4ef98e4b9f52cadbe49b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b 2024-12-11T06:16:37,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b, entries=200, sequenceid=544, filesize=14.4 K 2024-12-11T06:16:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/464a8bd2f2f448f4b0bd3e3c0c0e682c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c 2024-12-11T06:16:37,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c, entries=150, sequenceid=544, filesize=12.0 K 2024-12-11T06:16:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/271810f76f1343e4968894850f83a559 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559 2024-12-11T06:16:37,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559, entries=150, sequenceid=544, filesize=12.0 K 2024-12-11T06:16:37,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 520aa7289fa4bbc233d80abb04a82df6 in 609ms, sequenceid=544, compaction requested=true 2024-12-11T06:16:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:37,126 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:37,126 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:37,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 520aa7289fa4bbc233d80abb04a82df6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:37,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:37,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:37,129 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:37,129 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/B is initiating minor compaction (all files) 2024-12-11T06:16:37,129 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/A is initiating minor compaction (all files) 2024-12-11T06:16:37,129 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/A in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,129 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/B in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,129 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/30d21bd5f4cc43abb757d57e98d43239, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=39.1 K 2024-12-11T06:16:37,129 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b8810c0847044ba7bcac42e6fd24921b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.7 K 2024-12-11T06:16:37,134 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d21bd5f4cc43abb757d57e98d43239, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:37,134 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b8810c0847044ba7bcac42e6fd24921b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:37,134 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37070520a99147f6a86a1c62417a2cf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897794673 2024-12-11T06:16:37,134 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting de1cc744214946fab5d9f36365ad959b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897794673 2024-12-11T06:16:37,135 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 464a8bd2f2f448f4b0bd3e3c0c0e682c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733897795863 2024-12-11T06:16:37,135 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6403590312a4ef98e4b9f52cadbe49b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733897795853 2024-12-11T06:16:37,155 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:37,155 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/854a33f702cf40368fe78750c0265012 is 50, key is test_row_0/B:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:37,164 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:37,165 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/3358dfa892994a3cab7918396a59d3c5 is 50, key is test_row_0/A:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:37,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742319_1495 (size=13085) 2024-12-11T06:16:37,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742320_1496 (size=13085) 2024-12-11T06:16:37,190 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/854a33f702cf40368fe78750c0265012 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/854a33f702cf40368fe78750c0265012 2024-12-11T06:16:37,196 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/B of 520aa7289fa4bbc233d80abb04a82df6 into 854a33f702cf40368fe78750c0265012(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:37,196 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:37,197 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/B, priority=13, startTime=1733897797126; duration=0sec 2024-12-11T06:16:37,197 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:37,197 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:B 2024-12-11T06:16:37,197 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:37,197 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/3358dfa892994a3cab7918396a59d3c5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/3358dfa892994a3cab7918396a59d3c5 2024-12-11T06:16:37,198 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:37,198 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 520aa7289fa4bbc233d80abb04a82df6/C is initiating minor compaction (all files) 2024-12-11T06:16:37,198 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 520aa7289fa4bbc233d80abb04a82df6/C in TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,198 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/101bdee5ec4a4e5a8370fe1ba64cda4f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp, totalSize=36.7 K 2024-12-11T06:16:37,199 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 101bdee5ec4a4e5a8370fe1ba64cda4f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733897794301 2024-12-11T06:16:37,199 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2acd225caf3a479497a3d3e34b837744, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733897794673 2024-12-11T06:16:37,200 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 271810f76f1343e4968894850f83a559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733897795863 2024-12-11T06:16:37,203 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/A of 520aa7289fa4bbc233d80abb04a82df6 into 3358dfa892994a3cab7918396a59d3c5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:37,203 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:37,203 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/A, priority=13, startTime=1733897797126; duration=0sec 2024-12-11T06:16:37,203 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:37,203 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:A 2024-12-11T06:16:37,210 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 520aa7289fa4bbc233d80abb04a82df6#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:37,210 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/cf75eb3c8cd44bb9b890507dcb4fa380 is 50, key is test_row_0/C:col10/1733897795878/Put/seqid=0 2024-12-11T06:16:37,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742321_1497 (size=13085) 2024-12-11T06:16:37,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:37,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:37,244 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/cf75eb3c8cd44bb9b890507dcb4fa380 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/cf75eb3c8cd44bb9b890507dcb4fa380 2024-12-11T06:16:37,252 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 520aa7289fa4bbc233d80abb04a82df6/C of 520aa7289fa4bbc233d80abb04a82df6 into cf75eb3c8cd44bb9b890507dcb4fa380(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:37,252 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:37,252 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., storeName=520aa7289fa4bbc233d80abb04a82df6/C, priority=13, startTime=1733897797126; duration=0sec 2024-12-11T06:16:37,252 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:37,252 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 520aa7289fa4bbc233d80abb04a82df6:C 2024-12-11T06:16:37,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/33bcbf68a88044679d7aad6426fc0a39 is 50, key is test_row_0/A:col10/1733897796570/Put/seqid=0 2024-12-11T06:16:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742322_1498 (size=14741) 2024-12-11T06:16:37,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/33bcbf68a88044679d7aad6426fc0a39 2024-12-11T06:16:37,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/367e37aa38014caaae2f67c315f2dc23 is 50, key is test_row_0/B:col10/1733897796570/Put/seqid=0 2024-12-11T06:16:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897857295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897857299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897857303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T06:16:37,319 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:37,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-11T06:16:37,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:37,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:37,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:37,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742323_1499 (size=12301) 2024-12-11T06:16:37,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/367e37aa38014caaae2f67c315f2dc23 2024-12-11T06:16:37,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/e371a6eebbd144959fcaad29604ca5d0 is 50, key is test_row_0/C:col10/1733897796570/Put/seqid=0 2024-12-11T06:16:37,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742324_1500 (size=12301) 2024-12-11T06:16:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897857419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897857420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897857422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:37,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T06:16:37,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T06:16:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44344 deadline: 1733897857628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44258 deadline: 1733897857628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1733897857629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 , the old value is region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., hostname=4518703b2c1d,35425,1733897658824, seqNum=2 because the exception is null or not the one we care about 2024-12-11T06:16:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:37,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/e371a6eebbd144959fcaad29604ca5d0 2024-12-11T06:16:37,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/33bcbf68a88044679d7aad6426fc0a39 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/33bcbf68a88044679d7aad6426fc0a39 2024-12-11T06:16:37,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/33bcbf68a88044679d7aad6426fc0a39, entries=200, sequenceid=563, filesize=14.4 K 2024-12-11T06:16:37,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/367e37aa38014caaae2f67c315f2dc23 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/367e37aa38014caaae2f67c315f2dc23 2024-12-11T06:16:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/367e37aa38014caaae2f67c315f2dc23, entries=150, sequenceid=563, filesize=12.0 K 2024-12-11T06:16:37,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/e371a6eebbd144959fcaad29604ca5d0 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/e371a6eebbd144959fcaad29604ca5d0 2024-12-11T06:16:37,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/e371a6eebbd144959fcaad29604ca5d0, entries=150, sequenceid=563, filesize=12.0 K 2024-12-11T06:16:37,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 520aa7289fa4bbc233d80abb04a82df6 in 541ms, sequenceid=563, compaction requested=false 2024-12-11T06:16:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:37,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T06:16:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:37,785 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:37,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/598e2438acdb43609065539007cb4a5a is 50, key is test_row_0/A:col10/1733897797292/Put/seqid=0 2024-12-11T06:16:37,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742325_1501 (size=12301) 2024-12-11T06:16:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:37,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. as already flushing 2024-12-11T06:16:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:37,963 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-12-11T06:16:37,963 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-12-11T06:16:37,963 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,963 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,963 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,963 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,963 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,963 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,963 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,965 INFO [Thread-1922 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1922. 2024-12-11T06:16:37,965 DEBUG [Thread-1922 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,965 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,965 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,971 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-12-11T06:16:37,971 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,971 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,971 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,973 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-12-11T06:16:37,973 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,974 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,974 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-12-11T06:16:37,974 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,974 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-12-11T06:16:37,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,974 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,974 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:37,978 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-12-11T06:16:37,978 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:37,978 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,978 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:37,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:38,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/598e2438acdb43609065539007cb4a5a 2024-12-11T06:16:38,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/40487d3e31e74da9bcf4b1af26f9e176 is 50, key is test_row_0/B:col10/1733897797292/Put/seqid=0 2024-12-11T06:16:38,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742326_1502 (size=12301) 2024-12-11T06:16:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:38,611 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/40487d3e31e74da9bcf4b1af26f9e176 2024-12-11T06:16:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/29f6132df3e547c4b65f15307c06554d is 50, key is test_row_0/C:col10/1733897797292/Put/seqid=0 2024-12-11T06:16:38,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742327_1503 (size=12301) 2024-12-11T06:16:39,025 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/29f6132df3e547c4b65f15307c06554d 2024-12-11T06:16:39,032 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-12-11T06:16:39,032 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:39,032 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,032 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/598e2438acdb43609065539007cb4a5a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/598e2438acdb43609065539007cb4a5a 2024-12-11T06:16:39,037 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/598e2438acdb43609065539007cb4a5a, entries=150, sequenceid=579, filesize=12.0 K 2024-12-11T06:16:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/40487d3e31e74da9bcf4b1af26f9e176 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/40487d3e31e74da9bcf4b1af26f9e176 2024-12-11T06:16:39,041 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-12-11T06:16:39,041 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:16:39,041 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,041 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:39,042 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/40487d3e31e74da9bcf4b1af26f9e176, entries=150, sequenceid=579, filesize=12.0 K 2024-12-11T06:16:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/29f6132df3e547c4b65f15307c06554d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/29f6132df3e547c4b65f15307c06554d 2024-12-11T06:16:39,046 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/29f6132df3e547c4b65f15307c06554d, entries=150, sequenceid=579, filesize=12.0 K 2024-12-11T06:16:39,046 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 520aa7289fa4bbc233d80abb04a82df6 in 1262ms, sequenceid=579, compaction requested=true 2024-12-11T06:16:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 520aa7289fa4bbc233d80abb04a82df6: 2024-12-11T06:16:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:39,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-11T06:16:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-11T06:16:39,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-11T06:16:39,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7260 sec 2024-12-11T06:16:39,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7310 sec 2024-12-11T06:16:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T06:16:39,469 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1324 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3961 rows 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1325 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3966 rows 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1338 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4014 rows 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1324 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3972 rows 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1268 2024-12-11T06:16:39,469 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3804 rows 2024-12-11T06:16:39,470 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:16:39,470 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:16:39,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:39,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:16:39,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:16:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:39,473 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897799473"}]},"ts":"1733897799473"} 2024-12-11T06:16:39,474 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:16:39,474 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:16:39,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:16:39,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, UNASSIGN}] 2024-12-11T06:16:39,477 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, UNASSIGN 2024-12-11T06:16:39,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=520aa7289fa4bbc233d80abb04a82df6, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:39,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, UNASSIGN because future has completed 2024-12-11T06:16:39,479 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:16:39,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:39,631 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:39,631 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:16:39,631 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 520aa7289fa4bbc233d80abb04a82df6, disabling compactions & flushes 2024-12-11T06:16:39,631 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:39,631 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:39,631 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. after waiting 0 ms 2024-12-11T06:16:39,631 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:39,632 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 520aa7289fa4bbc233d80abb04a82df6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=A 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=B 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 520aa7289fa4bbc233d80abb04a82df6, store=C 2024-12-11T06:16:39,632 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:39,636 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/38ac4584cc2d4a6e8619bb414272aecc is 50, key is test_row_0/A:col10/1733897799039/Put/seqid=0 2024-12-11T06:16:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742328_1504 (size=12301) 2024-12-11T06:16:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:40,040 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/38ac4584cc2d4a6e8619bb414272aecc 2024-12-11T06:16:40,049 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5301b1055b454ee68efb5ba416d7ac88 is 50, key is test_row_0/B:col10/1733897799039/Put/seqid=0 2024-12-11T06:16:40,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742329_1505 (size=12301) 2024-12-11T06:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:40,454 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5301b1055b454ee68efb5ba416d7ac88 2024-12-11T06:16:40,466 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/0fd4e26cacda4d33bc55249e4cbbbb2a is 50, key is test_row_0/C:col10/1733897799039/Put/seqid=0 2024-12-11T06:16:40,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742330_1506 (size=12301) 2024-12-11T06:16:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:40,871 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/0fd4e26cacda4d33bc55249e4cbbbb2a 2024-12-11T06:16:40,879 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/A/38ac4584cc2d4a6e8619bb414272aecc as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/38ac4584cc2d4a6e8619bb414272aecc 2024-12-11T06:16:40,885 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/38ac4584cc2d4a6e8619bb414272aecc, entries=150, sequenceid=596, filesize=12.0 K 2024-12-11T06:16:40,886 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/B/5301b1055b454ee68efb5ba416d7ac88 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5301b1055b454ee68efb5ba416d7ac88 2024-12-11T06:16:40,890 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5301b1055b454ee68efb5ba416d7ac88, entries=150, sequenceid=596, filesize=12.0 K 2024-12-11T06:16:40,891 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/.tmp/C/0fd4e26cacda4d33bc55249e4cbbbb2a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/0fd4e26cacda4d33bc55249e4cbbbb2a 2024-12-11T06:16:40,895 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/0fd4e26cacda4d33bc55249e4cbbbb2a, entries=150, sequenceid=596, filesize=12.0 K 2024-12-11T06:16:40,896 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 520aa7289fa4bbc233d80abb04a82df6 in 1265ms, sequenceid=596, compaction requested=true 2024-12-11T06:16:40,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/06d422edcf8c4b8a86a54e42c4046801, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c9b5a6eea0504099ad7f551d98d5c79b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a3d39e8ca4b1499c8d66e64cf7798bde, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/48005924c2de412f96876968f5dd8be9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/b7c4b3bbdccf479283446a4b67eccbcb, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/30d21bd5f4cc43abb757d57e98d43239, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b] to archive 2024-12-11T06:16:40,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:40,901 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/6dca3cf023404090a8773805bec1cdba 2024-12-11T06:16:40,901 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/97f898f3120e41b5960b421674128797 2024-12-11T06:16:40,901 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/647855a2970c4b4bb51e76a023cd0709 2024-12-11T06:16:40,901 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/06d422edcf8c4b8a86a54e42c4046801 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/06d422edcf8c4b8a86a54e42c4046801 2024-12-11T06:16:40,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/914d2064d4d34c78b8837f190a3febb6 2024-12-11T06:16:40,902 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/71de6642627d49069974a942df6458d1 2024-12-11T06:16:40,902 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/360b1dd284ad4255a64e55b6cea99181 2024-12-11T06:16:40,902 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c9b5a6eea0504099ad7f551d98d5c79b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c9b5a6eea0504099ad7f551d98d5c79b 2024-12-11T06:16:40,903 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/e979f713c6de44a184f990771e14786e 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/65009073a27142d99973a553056cb4a9 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/24543bad04ec4c1d9e3f60a80ccbc697 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/4bb4ba0fbdca4807ac8fdca2234f8e78 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a3d39e8ca4b1499c8d66e64cf7798bde to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a3d39e8ca4b1499c8d66e64cf7798bde 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/c17c6b540ab24d91b30f2dcfe09c90b4 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/48005924c2de412f96876968f5dd8be9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/48005924c2de412f96876968f5dd8be9 2024-12-11T06:16:40,904 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/5c4107f7f4de44bd977b255f18a45963 2024-12-11T06:16:40,905 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0739e1f4bed947a9989b9b300c3cd209 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/0eea2f6059d04bee9d435e09a27aa1a8 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/837a2039eb804e7eb241e0b21ff1f5e2 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/b7c4b3bbdccf479283446a4b67eccbcb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/b7c4b3bbdccf479283446a4b67eccbcb 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/30d21bd5f4cc43abb757d57e98d43239 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/30d21bd5f4cc43abb757d57e98d43239 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/37070520a99147f6a86a1c62417a2cf8 2024-12-11T06:16:40,906 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/a6403590312a4ef98e4b9f52cadbe49b 2024-12-11T06:16:40,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/c01e831ae3ca4176a9fc597bc2d9c454, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ce67a5ec22954197bc310dfbf97ce72c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/6ae0a65da7e14a58a2073905400b6212, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9119b4212e6e4704a8c8543e36e177b0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/3490b8b995834e509167ac6f981d367b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b8810c0847044ba7bcac42e6fd24921b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c] to archive 2024-12-11T06:16:40,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b956318618894a7183538ca5e35d935b 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b0f6fb5673ca4365a92f6fcb97e59aa0 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/c01e831ae3ca4176a9fc597bc2d9c454 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/c01e831ae3ca4176a9fc597bc2d9c454 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/7a6cf03912e04e679f07006ac584eb6b 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ce67a5ec22954197bc310dfbf97ce72c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ce67a5ec22954197bc310dfbf97ce72c 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9386b359ea4d4d8e8245c502d167e7bd 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/d9b056102b9b45d18cb98781e5f8eb13 2024-12-11T06:16:40,910 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/216e8da20ed240009da23266e8b199c9 2024-12-11T06:16:40,911 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/dba27c01f2394cab8bf50d7b7a3aa20a 2024-12-11T06:16:40,911 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/311c886a757346e3b814c2d1d440a7f1 2024-12-11T06:16:40,911 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/8cd78ac0b648413984323faabf8bf38e 2024-12-11T06:16:40,911 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5968bea3a92346ffa7e269fc7c44436a 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/6ae0a65da7e14a58a2073905400b6212 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/6ae0a65da7e14a58a2073905400b6212 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9119b4212e6e4704a8c8543e36e177b0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/9119b4212e6e4704a8c8543e36e177b0 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/ec5501b73f2043b1a75cba083d549173 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/97797d75ae30459f919156aa1c5ced0c 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/3490b8b995834e509167ac6f981d367b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/3490b8b995834e509167ac6f981d367b 2024-12-11T06:16:40,912 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/a201632ac1a946a1ac9f62bfe9028029 2024-12-11T06:16:40,913 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/566ce1bd618c48b098c001ef938f6ba6 2024-12-11T06:16:40,913 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b8810c0847044ba7bcac42e6fd24921b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/b8810c0847044ba7bcac42e6fd24921b 2024-12-11T06:16:40,913 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/2a0df90acf8c421f98068121e2e4fe2e 2024-12-11T06:16:40,913 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/464a8bd2f2f448f4b0bd3e3c0c0e682c 2024-12-11T06:16:40,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/de1cc744214946fab5d9f36365ad959b 2024-12-11T06:16:40,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/10d2ff65215941449665092e9825d8dc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/117024a8e3974a4d9b7bd3c014901143, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/c86b112236314f139eef5b626ca32cf2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/112960f694ef47e2a1784ee0549ef64e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/458565b1bcde4e50b5c542500a923867, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/101bdee5ec4a4e5a8370fe1ba64cda4f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559] to archive 2024-12-11T06:16:40,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:16:40,915 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/74ad4532ebe24c389237dcceaa8e29e5 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/6ddfc748b04643c389a76478576d8cb0 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ebbc9f5fc71b4c2eb98d16dbc9ee2098 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/12297077364c471c9382445aea18dde6 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/117024a8e3974a4d9b7bd3c014901143 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/117024a8e3974a4d9b7bd3c014901143 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/9a0615bbf19140d894793a631eca3936 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/10d2ff65215941449665092e9825d8dc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/10d2ff65215941449665092e9825d8dc 2024-12-11T06:16:40,916 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/b722496ae7c044deb6825e0a2c370b65 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/fbafab28b5ea45e3b5adfb5e5b51e7f3 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/355bc65ea43547c986776741d2a13cce 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/a93019a6434d44b38169cbd485938d3c 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ade4283c235d4603bdbb670e9da90b64 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/eccc3bbc572f492ab9cf93ab285dd80e 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/c86b112236314f139eef5b626ca32cf2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/c86b112236314f139eef5b626ca32cf2 2024-12-11T06:16:40,917 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/112960f694ef47e2a1784ee0549ef64e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/112960f694ef47e2a1784ee0549ef64e 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/22ee1d017b084d4fb29b03ee549952dc 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/ee09cb21a476458680ddc9ae1df4f481 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/d4fed49a692c45d8b90d3ac7874da767 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/101bdee5ec4a4e5a8370fe1ba64cda4f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/101bdee5ec4a4e5a8370fe1ba64cda4f 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/458565b1bcde4e50b5c542500a923867 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/458565b1bcde4e50b5c542500a923867 2024-12-11T06:16:40,918 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/2acd225caf3a479497a3d3e34b837744 2024-12-11T06:16:40,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/63eec9ca7e174c8fa03c7032e61c475b 2024-12-11T06:16:40,919 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/271810f76f1343e4968894850f83a559 2024-12-11T06:16:40,923 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/recovered.edits/599.seqid, newMaxSeqId=599, maxSeqId=1 2024-12-11T06:16:40,924 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. 2024-12-11T06:16:40,924 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 520aa7289fa4bbc233d80abb04a82df6: Waiting for close lock at 1733897799631Running coprocessor pre-close hooks at 1733897799631Disabling compacts and flushes for region at 1733897799631Disabling writes for close at 1733897799631Obtaining lock to block concurrent updates at 1733897799632 (+1 ms)Preparing flush snapshotting stores in 520aa7289fa4bbc233d80abb04a82df6 at 1733897799632Finished memstore snapshotting TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6., syncing WAL and waiting on mvcc, flushsize=dataSize=96180, getHeapSize=286320, getOffHeapSize=0, getCellsCount=2100 at 1733897799632Flushing stores of TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6. at 1733897799633 (+1 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/A: creating writer at 1733897799633Flushing 520aa7289fa4bbc233d80abb04a82df6/A: appending metadata at 1733897799636 (+3 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/A: closing flushed file at 1733897799636Flushing 520aa7289fa4bbc233d80abb04a82df6/B: creating writer at 1733897800045 (+409 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/B: appending metadata at 1733897800048 (+3 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/B: closing flushed file at 1733897800048Flushing 520aa7289fa4bbc233d80abb04a82df6/C: creating writer at 1733897800461 (+413 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/C: appending metadata at 1733897800465 (+4 ms)Flushing 520aa7289fa4bbc233d80abb04a82df6/C: closing flushed file at 1733897800465Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33dc4041: reopening flushed file at 1733897800878 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f25a821: reopening flushed file at 1733897800885 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c081ae0: reopening flushed file at 1733897800890 (+5 ms)Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 520aa7289fa4bbc233d80abb04a82df6 in 1265ms, sequenceid=596, compaction requested=true at 1733897800896 (+6 ms)Writing region close event to WAL at 1733897800921 (+25 ms)Running coprocessor post-close hooks at 1733897800923 (+2 ms)Closed at 1733897800924 (+1 ms) 2024-12-11T06:16:40,925 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:40,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=520aa7289fa4bbc233d80abb04a82df6, regionState=CLOSED 2024-12-11T06:16:40,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:40,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-11T06:16:40,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 520aa7289fa4bbc233d80abb04a82df6, server=4518703b2c1d,35425,1733897658824 in 1.4480 sec 2024-12-11T06:16:40,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-11T06:16:40,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=520aa7289fa4bbc233d80abb04a82df6, UNASSIGN in 1.4520 sec 2024-12-11T06:16:40,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-11T06:16:40,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4550 sec 2024-12-11T06:16:40,932 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897800932"}]},"ts":"1733897800932"} 2024-12-11T06:16:40,933 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:16:40,933 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:16:40,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4630 sec 2024-12-11T06:16:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T06:16:41,620 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:41,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:16:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,626 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T06:16:41,628 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,631 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:41,634 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/recovered.edits] 2024-12-11T06:16:41,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/33bcbf68a88044679d7aad6426fc0a39 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/33bcbf68a88044679d7aad6426fc0a39 2024-12-11T06:16:41,637 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/598e2438acdb43609065539007cb4a5a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/598e2438acdb43609065539007cb4a5a 2024-12-11T06:16:41,637 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/3358dfa892994a3cab7918396a59d3c5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/3358dfa892994a3cab7918396a59d3c5 2024-12-11T06:16:41,638 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/38ac4584cc2d4a6e8619bb414272aecc to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/A/38ac4584cc2d4a6e8619bb414272aecc 2024-12-11T06:16:41,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/367e37aa38014caaae2f67c315f2dc23 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/367e37aa38014caaae2f67c315f2dc23 2024-12-11T06:16:41,641 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/40487d3e31e74da9bcf4b1af26f9e176 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/40487d3e31e74da9bcf4b1af26f9e176 2024-12-11T06:16:41,642 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5301b1055b454ee68efb5ba416d7ac88 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/5301b1055b454ee68efb5ba416d7ac88 2024-12-11T06:16:41,642 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/854a33f702cf40368fe78750c0265012 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/B/854a33f702cf40368fe78750c0265012 2024-12-11T06:16:41,646 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/cf75eb3c8cd44bb9b890507dcb4fa380 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/cf75eb3c8cd44bb9b890507dcb4fa380 2024-12-11T06:16:41,646 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/0fd4e26cacda4d33bc55249e4cbbbb2a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/0fd4e26cacda4d33bc55249e4cbbbb2a 2024-12-11T06:16:41,646 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/29f6132df3e547c4b65f15307c06554d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/29f6132df3e547c4b65f15307c06554d 2024-12-11T06:16:41,646 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/e371a6eebbd144959fcaad29604ca5d0 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/C/e371a6eebbd144959fcaad29604ca5d0 2024-12-11T06:16:41,649 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/recovered.edits/599.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6/recovered.edits/599.seqid 2024-12-11T06:16:41,650 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/520aa7289fa4bbc233d80abb04a82df6 2024-12-11T06:16:41,650 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:16:41,653 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,655 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:16:41,658 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:16:41,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:16:41,659 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897801659"}]},"ts":"9223372036854775807"} 2024-12-11T06:16:41,661 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:16:41,661 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 520aa7289fa4bbc233d80abb04a82df6, NAME => 'TestAcidGuarantees,,1733897776574.520aa7289fa4bbc233d80abb04a82df6.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:16:41,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:16:41,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897801661"}]},"ts":"9223372036854775807"} 2024-12-11T06:16:41,663 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:16:41,663 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:41,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-11T06:16:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T06:16:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:16:41,746 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:42,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255 (was 255), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=837 (was 891), ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 533) 2024-12-11T06:16:42,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=837, ProcessCount=11, AvailableMemoryMB=238 2024-12-11T06:16:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:16:42,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:16:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:42,040 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T06:16:42,040 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:42,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-12-11T06:16:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:42,041 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T06:16:42,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742331_1507 (size=1061) 2024-12-11T06:16:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:42,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915 2024-12-11T06:16:42,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742332_1508 (size=53) 2024-12-11T06:16:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:42,861 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:16:42,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9c9700b86d470c316ec43a15e9141439, disabling compactions & flushes 2024-12-11T06:16:42,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:42,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:42,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. after waiting 0 ms 2024-12-11T06:16:42,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:42,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:42,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c9700b86d470c316ec43a15e9141439: Waiting for close lock at 1733897802862Disabling compacts and flushes for region at 1733897802862Disabling writes for close at 1733897802862Writing region close event to WAL at 1733897802862Closed at 1733897802862 2024-12-11T06:16:42,865 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T06:16:42,866 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733897802865"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733897802865"}]},"ts":"1733897802865"} 2024-12-11T06:16:42,868 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T06:16:42,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T06:16:42,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897802869"}]},"ts":"1733897802869"} 2024-12-11T06:16:42,871 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T06:16:42,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, ASSIGN}] 2024-12-11T06:16:42,872 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, ASSIGN 2024-12-11T06:16:42,873 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, ASSIGN; state=OFFLINE, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=false 2024-12-11T06:16:43,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:43,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, ASSIGN because future has completed 2024-12-11T06:16:43,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:43,183 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:43,183 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:16:43,183 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,183 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:16:43,183 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,183 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,185 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,187 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:43,187 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName A 2024-12-11T06:16:43,187 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:43,188 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:43,188 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,189 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:43,190 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName B 2024-12-11T06:16:43,190 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:43,190 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:43,190 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,192 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:43,192 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName C 2024-12-11T06:16:43,192 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:43,192 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:43,193 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:43,193 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,194 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,194 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,195 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,195 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,196 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:16:43,196 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,198 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T06:16:43,198 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 9c9700b86d470c316ec43a15e9141439; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74506486, jitterRate=0.11023315787315369}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:16:43,198 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:43,199 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 9c9700b86d470c316ec43a15e9141439: Running coprocessor pre-open hook at 1733897803184Writing region info on filesystem at 1733897803184Initializing all the Stores at 1733897803184Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897803185 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897803185Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897803185Cleaning up temporary data from old regions at 1733897803195 (+10 ms)Running coprocessor post-open hooks at 1733897803198 (+3 ms)Region opened successfully at 1733897803198 2024-12-11T06:16:43,199 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., pid=136, masterSystemTime=1733897803179 2024-12-11T06:16:43,200 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:43,200 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:43,201 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=OPEN, openSeqNum=2, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:43,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:43,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-11T06:16:43,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 in 176 msec 2024-12-11T06:16:43,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-11T06:16:43,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, ASSIGN in 332 msec 2024-12-11T06:16:43,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T06:16:43,206 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897803205"}]},"ts":"1733897803205"} 2024-12-11T06:16:43,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T06:16:43,207 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T06:16:43,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-12-11T06:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-11T06:16:44,189 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:44,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363fa589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:44,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:44,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:44,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7deda434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:44,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cddc693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:44,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:44,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:44,198 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T06:16:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4518703b2c1d,41397,1733897657727 2024-12-11T06:16:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@753f721f 2024-12-11T06:16:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T06:16:44,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T06:16:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T06:16:44,201 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T06:16:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T06:16:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742333_1509 (size=1097) 2024-12-11T06:16:44,614 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T06:16:44,614 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T06:16:44,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:16:44,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE}] 2024-12-11T06:16:44,623 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE 2024-12-11T06:16:44,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:44,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE because future has completed 2024-12-11T06:16:44,625 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:16:44,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:44,778 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:16:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 9c9700b86d470c316ec43a15e9141439, disabling compactions & flushes 2024-12-11T06:16:44,778 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. after waiting 0 ms 2024-12-11T06:16:44,778 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:44,784 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T06:16:44,785 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:44,785 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 9c9700b86d470c316ec43a15e9141439: Waiting for close lock at 1733897804778Running coprocessor pre-close hooks at 1733897804778Disabling compacts and flushes for region at 1733897804778Disabling writes for close at 1733897804778Writing region close event to WAL at 1733897804779 (+1 ms)Running coprocessor post-close hooks at 1733897804785 (+6 ms)Closed at 1733897804785 2024-12-11T06:16:44,785 WARN [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 9c9700b86d470c316ec43a15e9141439 to self. 2024-12-11T06:16:44,788 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:44,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=CLOSED 2024-12-11T06:16:44,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:44,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-11T06:16:44,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 in 167 msec 2024-12-11T06:16:44,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE; state=CLOSED, location=4518703b2c1d,35425,1733897658824; forceNewPlan=false, retain=true 2024-12-11T06:16:44,946 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=OPENING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:44,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE because future has completed 2024-12-11T06:16:44,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:16:45,105 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,105 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} 2024-12-11T06:16:45,106 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,106 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T06:16:45,106 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,106 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,107 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,108 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:45,108 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName A 2024-12-11T06:16:45,109 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:45,110 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:45,110 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,110 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:45,111 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName B 2024-12-11T06:16:45,111 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:45,111 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:45,111 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,112 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T06:16:45,112 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c9700b86d470c316ec43a15e9141439 columnFamilyName C 2024-12-11T06:16:45,112 DEBUG [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:45,113 INFO [StoreOpener-9c9700b86d470c316ec43a15e9141439-1 {}] regionserver.HStore(327): Store=9c9700b86d470c316ec43a15e9141439/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T06:16:45,113 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,113 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,114 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,115 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,116 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,116 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,117 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T06:16:45,118 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,119 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 9c9700b86d470c316ec43a15e9141439; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63492489, jitterRate=-0.05388818681240082}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T06:16:45,119 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,120 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 9c9700b86d470c316ec43a15e9141439: Running coprocessor pre-open hook at 1733897805106Writing region info on filesystem at 1733897805106Initializing all the Stores at 1733897805107 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897805107Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897805107Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733897805107Cleaning up temporary data from old regions at 1733897805116 (+9 ms)Running coprocessor post-open hooks at 1733897805119 (+3 ms)Region opened successfully at 1733897805120 (+1 ms) 2024-12-11T06:16:45,120 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., pid=141, masterSystemTime=1733897805103 2024-12-11T06:16:45,122 DEBUG [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,122 INFO [RS_OPEN_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,123 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=OPEN, openSeqNum=5, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:16:45,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-12-11T06:16:45,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 in 175 msec 2024-12-11T06:16:45,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-11T06:16:45,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, REOPEN/MOVE in 506 msec 2024-12-11T06:16:45,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-11T06:16:45,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-12-11T06:16:45,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-12-11T06:16:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T06:16:45,136 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:45,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab33ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,139 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2335daa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5917614a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,143 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acb8e4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,146 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b16922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125b6dfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2f3bb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a1935b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c2a6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c6b101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db404f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c89c532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a7434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4521cf81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150c6132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40714fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,164 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7160d4d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66be6e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,167 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcd392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,168 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0d9c95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742259fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,170 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff19a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,171 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4dbf22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302eb4d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4518703b2c1d,41397,-1 for getting cluster id 2024-12-11T06:16:45,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T06:16:45,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7644275-0103-4d98-b05f-d576422ffe05' 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7644275-0103-4d98-b05f-d576422ffe05" 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb71a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4518703b2c1d,41397,-1] 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T06:16:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:16:45,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T06:16:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365089cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T06:16:45,175 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,176 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,176 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,177 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T06:16:45,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-11T06:16:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4518703b2c1d,35425,1733897658824, seqNum=-1] 2024-12-11T06:16:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T06:16:45,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T06:16:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5] 2024-12-11T06:16:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:45,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897805198, max timestamp=1733897805205Num uniques 859; 2024-12-11T06:16:45,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733897805198, max timestamp=1733897805205Num uniques 859; 2024-12-11T06:16:45,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:16:45,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T06:16:45,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897805198, max timestamp=1733897805205Num uniques 859; 2024-12-11T06:16:45,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:45,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f5b967678cfb484896847b6750b19ab7_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897805187/Put/seqid=0 2024-12-11T06:16:45,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742334_1510 (size=12154) 2024-12-11T06:16:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897865222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897865223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897865224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897865225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897865225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:45,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897865328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:45,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:45,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897865338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897865339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897865339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897865339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:45,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:45,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897865538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897865549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897865549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897865549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897865549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,624 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:45,627 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f5b967678cfb484896847b6750b19ab7_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f5b967678cfb484896847b6750b19ab7_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:45,628 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/754ad2173e224632892ebcc2194ecb0d, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:45,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/754ad2173e224632892ebcc2194ecb0d is 175, key is test_row_0/A:col10/1733897805187/Put/seqid=0 2024-12-11T06:16:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742335_1511 (size=30955) 2024-12-11T06:16:45,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:45,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:45,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:45,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897865848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897865858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897865859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897865859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897865859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:45,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:45,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/754ad2173e224632892ebcc2194ecb0d 2024-12-11T06:16:46,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3a4db23f0cdd4707a48a8e34fcb767ff is 50, key is test_row_0/B:col10/1733897805187/Put/seqid=0 2024-12-11T06:16:46,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742336_1512 (size=12001) 2024-12-11T06:16:46,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3a4db23f0cdd4707a48a8e34fcb767ff 2024-12-11T06:16:46,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ada5888d9dba441b9b521714523f4c4a is 50, key is test_row_0/C:col10/1733897805187/Put/seqid=0 2024-12-11T06:16:46,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742337_1513 (size=12001) 2024-12-11T06:16:46,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:46,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:46,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897866358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897866368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897866368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897866369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897866369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:46,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:46,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T06:16:46,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ada5888d9dba441b9b521714523f4c4a 2024-12-11T06:16:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/754ad2173e224632892ebcc2194ecb0d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d 2024-12-11T06:16:46,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d, entries=150, sequenceid=15, filesize=30.2 K 2024-12-11T06:16:46,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3a4db23f0cdd4707a48a8e34fcb767ff as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff 2024-12-11T06:16:46,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T06:16:46,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ada5888d9dba441b9b521714523f4c4a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a 2024-12-11T06:16:46,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T06:16:46,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 9c9700b86d470c316ec43a15e9141439 in 1309ms, sequenceid=15, compaction requested=false 2024-12-11T06:16:46,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:46,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-11T06:16:46,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:46,556 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-11T06:16:46,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121148c3a932f7d14fe4b815dd7ccf3ad9a4_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897805222/Put/seqid=0 2024-12-11T06:16:46,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742338_1514 (size=12154) 2024-12-11T06:16:46,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:46,570 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121148c3a932f7d14fe4b815dd7ccf3ad9a4_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121148c3a932f7d14fe4b815dd7ccf3ad9a4_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/906af02e3d8247d38d418bb53b331b7f, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/906af02e3d8247d38d418bb53b331b7f is 175, key is test_row_0/A:col10/1733897805222/Put/seqid=0 2024-12-11T06:16:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742339_1515 (size=30955) 2024-12-11T06:16:46,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T06:16:46,976 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/906af02e3d8247d38d418bb53b331b7f 2024-12-11T06:16:46,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/34c3713a84864526a881a63fa799c1f3 is 50, key is test_row_0/B:col10/1733897805222/Put/seqid=0 2024-12-11T06:16:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742340_1516 (size=12001) 2024-12-11T06:16:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:47,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897867380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897867381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,385 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/34c3713a84864526a881a63fa799c1f3 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897867382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897867383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897867389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,390 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/f2631dd34b9640a2b7aa464be0cc2a37 is 50, key is test_row_0/C:col10/1733897805222/Put/seqid=0 2024-12-11T06:16:47,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742341_1517 (size=12001) 2024-12-11T06:16:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897867489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897867489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897867489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897867489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897867698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897867698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897867699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897867699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:47,798 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/f2631dd34b9640a2b7aa464be0cc2a37 2024-12-11T06:16:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/906af02e3d8247d38d418bb53b331b7f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f 2024-12-11T06:16:47,804 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f, entries=150, sequenceid=51, filesize=30.2 K 2024-12-11T06:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/34c3713a84864526a881a63fa799c1f3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3 2024-12-11T06:16:47,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:16:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/f2631dd34b9640a2b7aa464be0cc2a37 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37 2024-12-11T06:16:47,811 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T06:16:47,811 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 9c9700b86d470c316ec43a15e9141439 in 1255ms, sequenceid=51, compaction requested=false 2024-12-11T06:16:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:47,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:47,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-11T06:16:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-11T06:16:47,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-11T06:16:47,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-12-11T06:16:47,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.6380 sec 2024-12-11T06:16:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:48,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:48,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:48,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412116b5d51681ef34252a6ed651334cbc7b7_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897868026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897868027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897868028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897868029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742342_1518 (size=17034) 2024-12-11T06:16:48,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897868138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897868138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897868138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897868138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:48,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412116b5d51681ef34252a6ed651334cbc7b7_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412116b5d51681ef34252a6ed651334cbc7b7_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:48,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/68220e35191c49dda49d0cf3e5968cad, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:48,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/68220e35191c49dda49d0cf3e5968cad is 175, key is test_row_0/A:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:48,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742343_1519 (size=48139) 2024-12-11T06:16:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897868659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897868659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897868659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897868659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:48,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/68220e35191c49dda49d0cf3e5968cad 2024-12-11T06:16:48,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1619464500ea4ad7ae8c36321c26ef22 is 50, key is test_row_0/B:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:48,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742344_1520 (size=12001) 2024-12-11T06:16:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897869168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897869168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897869169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897869169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1619464500ea4ad7ae8c36321c26ef22 2024-12-11T06:16:49,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/79cd5b7552924f369088ceecd48af2bf is 50, key is test_row_0/C:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:49,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742345_1521 (size=12001) 2024-12-11T06:16:49,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/79cd5b7552924f369088ceecd48af2bf 2024-12-11T06:16:49,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/68220e35191c49dda49d0cf3e5968cad as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad 2024-12-11T06:16:49,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad, entries=250, sequenceid=64, filesize=47.0 K 2024-12-11T06:16:49,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1619464500ea4ad7ae8c36321c26ef22 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22 2024-12-11T06:16:49,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22, entries=150, sequenceid=64, filesize=11.7 K 2024-12-11T06:16:49,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/79cd5b7552924f369088ceecd48af2bf as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf 2024-12-11T06:16:49,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf, entries=150, sequenceid=64, filesize=11.7 K 2024-12-11T06:16:49,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9c9700b86d470c316ec43a15e9141439 in 1290ms, sequenceid=64, compaction requested=true 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:16:49,301 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:16:49,301 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,301 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,302 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=107.5 K 2024-12-11T06:16:49,302 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.2 K 2024-12-11T06:16:49,302 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad] 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4db23f0cdd4707a48a8e34fcb767ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897805181 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754ad2173e224632892ebcc2194ecb0d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897805181 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c3713a84864526a881a63fa799c1f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897805222 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906af02e3d8247d38d418bb53b331b7f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897805222 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1619464500ea4ad7ae8c36321c26ef22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897808009 2024-12-11T06:16:49,302 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68220e35191c49dda49d0cf3e5968cad, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897807381 2024-12-11T06:16:49,307 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:49,308 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:49,308 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/cc976353f990442faecbe73cf63e8a15 is 50, key is test_row_0/B:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:49,321 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211e499ddb1d60d45e6aaeb2c77739a6aab_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:49,322 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211e499ddb1d60d45e6aaeb2c77739a6aab_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:49,323 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e499ddb1d60d45e6aaeb2c77739a6aab_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742346_1522 (size=12104) 2024-12-11T06:16:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742347_1523 (size=4469) 2024-12-11T06:16:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T06:16:49,338 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:49,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-11T06:16:49,340 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:49,340 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:49,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:49,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:49,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114926e508af304735ad2400757d4c17ab_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897808026/Put/seqid=0 2024-12-11T06:16:49,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742348_1524 (size=14594) 2024-12-11T06:16:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:49,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897869472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:49,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897869588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:49,729 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/cc976353f990442faecbe73cf63e8a15 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/cc976353f990442faecbe73cf63e8a15 2024-12-11T06:16:49,731 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#445 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:49,731 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2eddd57dcca44d46ac1d146d5e2d24b4 is 175, key is test_row_0/A:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:49,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742349_1525 (size=31058) 2024-12-11T06:16:49,734 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into cc976353f990442faecbe73cf63e8a15(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:49,734 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:49,734 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=13, startTime=1733897809301; duration=0sec 2024-12-11T06:16:49,734 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:49,734 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:16:49,734 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:49,735 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:49,735 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:16:49,735 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,735 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.2 K 2024-12-11T06:16:49,736 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ada5888d9dba441b9b521714523f4c4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733897805181 2024-12-11T06:16:49,736 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f2631dd34b9640a2b7aa464be0cc2a37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733897805222 2024-12-11T06:16:49,736 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 79cd5b7552924f369088ceecd48af2bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897808009 2024-12-11T06:16:49,741 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:49,742 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/0cc49ec7e63c49f980cf9ba884dcd2ca is 50, key is test_row_0/C:col10/1733897808009/Put/seqid=0 2024-12-11T06:16:49,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742350_1526 (size=12104) 2024-12-11T06:16:49,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897869799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:49,830 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:49,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114926e508af304735ad2400757d4c17ab_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114926e508af304735ad2400757d4c17ab_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:49,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/4d57742c9ce54dbda006e5660accd9e7, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:49,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/4d57742c9ce54dbda006e5660accd9e7 is 175, key is test_row_0/A:col10/1733897808026/Put/seqid=0 2024-12-11T06:16:49,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742351_1527 (size=39549) 2024-12-11T06:16:49,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:49,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:50,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897870108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,138 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2eddd57dcca44d46ac1d146d5e2d24b4 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4 2024-12-11T06:16:50,143 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into 2eddd57dcca44d46ac1d146d5e2d24b4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:50,143 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:50,143 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=13, startTime=1733897809301; duration=0sec 2024-12-11T06:16:50,143 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:50,143 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:16:50,149 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/0cc49ec7e63c49f980cf9ba884dcd2ca as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0cc49ec7e63c49f980cf9ba884dcd2ca 2024-12-11T06:16:50,153 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into 0cc49ec7e63c49f980cf9ba884dcd2ca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:50,153 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:50,153 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=13, startTime=1733897809301; duration=0sec 2024-12-11T06:16:50,153 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:50,153 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:16:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897870179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897870179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897870179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,181 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,181 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,182 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897870188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,190 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/4d57742c9ce54dbda006e5660accd9e7 2024-12-11T06:16:50,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3b89980f33c9427ca2d2e9b9045bb459 is 50, key is test_row_0/B:col10/1733897808026/Put/seqid=0 2024-12-11T06:16:50,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742352_1528 (size=12001) 2024-12-11T06:16:50,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3b89980f33c9427ca2d2e9b9045bb459 2024-12-11T06:16:50,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/027f6675312a4899bbd2c688a887e5d8 is 50, key is test_row_0/C:col10/1733897808026/Put/seqid=0 2024-12-11T06:16:50,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742353_1529 (size=12001) 2024-12-11T06:16:50,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:50,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:50,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:50,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897870618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:50,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/027f6675312a4899bbd2c688a887e5d8 2024-12-11T06:16:50,708 INFO [AsyncFSWAL-0-hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData-prefix:4518703b2c1d,41397,1733897657727 {}] wal.AbstractFSWAL(1368): Slow sync cost: 299 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33105,DS-73cb111b-9445-42c1-a84f-533c2b23174b,DISK]] 2024-12-11T06:16:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/4d57742c9ce54dbda006e5660accd9e7 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7 2024-12-11T06:16:50,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7, entries=200, sequenceid=84, filesize=38.6 K 2024-12-11T06:16:50,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3b89980f33c9427ca2d2e9b9045bb459 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459 2024-12-11T06:16:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459, entries=150, sequenceid=84, filesize=11.7 K 2024-12-11T06:16:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/027f6675312a4899bbd2c688a887e5d8 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8 2024-12-11T06:16:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8, entries=150, sequenceid=84, filesize=11.7 K 2024-12-11T06:16:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9c9700b86d470c316ec43a15e9141439 in 1300ms, sequenceid=84, compaction requested=false 2024-12-11T06:16:50,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-11T06:16:50,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,860 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113b65840d0c894e09b10a80aef174ba5f_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897809471/Put/seqid=0 2024-12-11T06:16:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742354_1530 (size=9714) 2024-12-11T06:16:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,878 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113b65840d0c894e09b10a80aef174ba5f_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113b65840d0c894e09b10a80aef174ba5f_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/5ef84b348bdd4c15b22973547bf5ee9e, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/5ef84b348bdd4c15b22973547bf5ee9e is 175, key is test_row_0/A:col10/1733897809471/Put/seqid=0 2024-12-11T06:16:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742355_1531 (size=22361) 2024-12-11T06:16:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,283 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/5ef84b348bdd4c15b22973547bf5ee9e 2024-12-11T06:16:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/dfd36b7286624950bd4c944ecf966ab9 is 50, key is test_row_0/B:col10/1733897809471/Put/seqid=0 2024-12-11T06:16:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742356_1532 (size=9657) 2024-12-11T06:16:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:51,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/dfd36b7286624950bd4c944ecf966ab9 2024-12-11T06:16:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e8df996522864c51ad31bc91240f184c is 50, key is test_row_0/C:col10/1733897809471/Put/seqid=0 2024-12-11T06:16:51,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742357_1533 (size=9657) 2024-12-11T06:16:51,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897871715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897871829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897872038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,102 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e8df996522864c51ad31bc91240f184c 2024-12-11T06:16:52,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/5ef84b348bdd4c15b22973547bf5ee9e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e 2024-12-11T06:16:52,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e, entries=100, sequenceid=99, filesize=21.8 K 2024-12-11T06:16:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/dfd36b7286624950bd4c944ecf966ab9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9 2024-12-11T06:16:52,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9, entries=100, sequenceid=99, filesize=9.4 K 2024-12-11T06:16:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e8df996522864c51ad31bc91240f184c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c 2024-12-11T06:16:52,114 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c, entries=100, sequenceid=99, filesize=9.4 K 2024-12-11T06:16:52,115 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9c9700b86d470c316ec43a15e9141439 in 1256ms, sequenceid=99, compaction requested=true 2024-12-11T06:16:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-11T06:16:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-11T06:16:52,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-11T06:16:52,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7760 sec 2024-12-11T06:16:52,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.7790 sec 2024-12-11T06:16:52,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:52,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f16b305b982045399fcc55b7adb77db3_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:52,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742358_1534 (size=14594) 2024-12-11T06:16:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897872213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897872214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897872214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897872218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,220 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897872319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897872328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897872328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897872348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897872529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897872539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897872539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:52,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f16b305b982045399fcc55b7adb77db3_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f16b305b982045399fcc55b7adb77db3_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:52,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/38bb6405f6c44fd08bdf7c2243fb2001, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:52,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/38bb6405f6c44fd08bdf7c2243fb2001 is 175, key is test_row_0/A:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:52,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742359_1535 (size=39549) 2024-12-11T06:16:52,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/38bb6405f6c44fd08bdf7c2243fb2001 2024-12-11T06:16:52,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/875a4ebeadd441f68984462ed6e877da is 50, key is test_row_0/B:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:52,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742360_1536 (size=12001) 2024-12-11T06:16:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897872838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897872848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897872848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:52,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:52,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897872858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:53,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/875a4ebeadd441f68984462ed6e877da 2024-12-11T06:16:53,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/8958869867a54c11b4ecee16a0a9d8f2 is 50, key is test_row_0/C:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:53,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742361_1537 (size=12001) 2024-12-11T06:16:53,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897873349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897873359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:53,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897873359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:53,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/8958869867a54c11b4ecee16a0a9d8f2 2024-12-11T06:16:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/38bb6405f6c44fd08bdf7c2243fb2001 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001 2024-12-11T06:16:53,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001, entries=200, sequenceid=120, filesize=38.6 K 2024-12-11T06:16:53,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/875a4ebeadd441f68984462ed6e877da as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da 2024-12-11T06:16:53,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da, entries=150, sequenceid=120, filesize=11.7 K 2024-12-11T06:16:53,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/8958869867a54c11b4ecee16a0a9d8f2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2 2024-12-11T06:16:53,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2, entries=150, sequenceid=120, filesize=11.7 K 2024-12-11T06:16:53,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 9c9700b86d470c316ec43a15e9141439 in 1252ms, sequenceid=120, compaction requested=true 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:53,442 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:53,442 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:53,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:16:53,443 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:53,443 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=129.4 K 2024-12-11T06:16:53,443 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001] 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:16:53,443 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eddd57dcca44d46ac1d146d5e2d24b4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897808009 2024-12-11T06:16:53,444 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:53,444 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/cc976353f990442faecbe73cf63e8a15, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=44.7 K 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d57742c9ce54dbda006e5660accd9e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897808026 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting cc976353f990442faecbe73cf63e8a15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897808009 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef84b348bdd4c15b22973547bf5ee9e, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733897809465 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38bb6405f6c44fd08bdf7c2243fb2001, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b89980f33c9427ca2d2e9b9045bb459, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897808026 2024-12-11T06:16:53,444 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd36b7286624950bd4c944ecf966ab9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733897809465 2024-12-11T06:16:53,445 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 875a4ebeadd441f68984462ed6e877da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:53,450 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:53,451 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:53,452 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bcfae8fc3d3f469788975d4092476bb9 is 50, key is test_row_0/B:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:53,453 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412114410e46d6855402ebd672420ebb0e695_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:53,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412114410e46d6855402ebd672420ebb0e695_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:53,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114410e46d6855402ebd672420ebb0e695_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:53,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742362_1538 (size=12241) 2024-12-11T06:16:53,461 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bcfae8fc3d3f469788975d4092476bb9 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcfae8fc3d3f469788975d4092476bb9 2024-12-11T06:16:53,465 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into bcfae8fc3d3f469788975d4092476bb9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:53,465 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:53,465 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=12, startTime=1733897813442; duration=0sec 2024-12-11T06:16:53,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742363_1539 (size=4469) 2024-12-11T06:16:53,465 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:53,465 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:16:53,466 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:16:53,466 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#457 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:53,466 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:16:53,467 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:16:53,467 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:53,467 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/43dd7ca6c0514b54ada27746a52d7b19 is 175, key is test_row_0/A:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:53,467 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0cc49ec7e63c49f980cf9ba884dcd2ca, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=44.7 K 2024-12-11T06:16:53,467 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc49ec7e63c49f980cf9ba884dcd2ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733897808009 2024-12-11T06:16:53,467 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 027f6675312a4899bbd2c688a887e5d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733897808026 2024-12-11T06:16:53,468 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e8df996522864c51ad31bc91240f184c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733897809465 2024-12-11T06:16:53,468 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 8958869867a54c11b4ecee16a0a9d8f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:53,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742364_1540 (size=31195) 2024-12-11T06:16:53,473 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:53,474 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/7595bcfb6b68418fb58403ff0f163de2 is 50, key is test_row_0/C:col10/1733897811705/Put/seqid=0 2024-12-11T06:16:53,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742365_1541 (size=12241) 2024-12-11T06:16:53,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-11T06:16:53,498 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:53,499 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-11T06:16:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:53,500 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:53,501 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:53,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:53,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:53,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:53,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121174d64ed3fd80476fbec8082720d809a5_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897812213/Put/seqid=0 2024-12-11T06:16:53,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742366_1542 (size=12254) 2024-12-11T06:16:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:53,873 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/43dd7ca6c0514b54ada27746a52d7b19 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19 2024-12-11T06:16:53,877 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into 43dd7ca6c0514b54ada27746a52d7b19(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:53,877 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:53,877 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=12, startTime=1733897813442; duration=0sec 2024-12-11T06:16:53,877 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:53,877 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:16:53,881 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/7595bcfb6b68418fb58403ff0f163de2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/7595bcfb6b68418fb58403ff0f163de2 2024-12-11T06:16:53,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:53,895 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into 7595bcfb6b68418fb58403ff0f163de2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:53,895 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:53,896 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=12, startTime=1733897813442; duration=0sec 2024-12-11T06:16:53,896 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:53,896 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:16:53,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897873937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897874049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:54,065 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121174d64ed3fd80476fbec8082720d809a5_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121174d64ed3fd80476fbec8082720d809a5_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/28e676174f0240fbad7b6406a662e81b, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:54,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/28e676174f0240fbad7b6406a662e81b is 175, key is test_row_0/A:col10/1733897812213/Put/seqid=0 2024-12-11T06:16:54,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742367_1543 (size=31055) 2024-12-11T06:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:54,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897874258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897874358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,359 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897874368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,370 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897874379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,381 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:54,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,470 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/28e676174f0240fbad7b6406a662e81b 2024-12-11T06:16:54,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/4024ce2f37e1462baa32b3e9f0c45582 is 50, key is test_row_0/B:col10/1733897812213/Put/seqid=0 2024-12-11T06:16:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742368_1544 (size=12101) 2024-12-11T06:16:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897874568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:54,887 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/4024ce2f37e1462baa32b3e9f0c45582 2024-12-11T06:16:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/77b79e559b2f48f9bcd18455f247f71b is 50, key is test_row_0/C:col10/1733897812213/Put/seqid=0 2024-12-11T06:16:54,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742369_1545 (size=12101) 2024-12-11T06:16:54,897 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/77b79e559b2f48f9bcd18455f247f71b 2024-12-11T06:16:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/28e676174f0240fbad7b6406a662e81b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b 2024-12-11T06:16:54,906 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b, entries=150, sequenceid=133, filesize=30.3 K 2024-12-11T06:16:54,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/4024ce2f37e1462baa32b3e9f0c45582 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582 2024-12-11T06:16:54,910 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582, entries=150, sequenceid=133, filesize=11.8 K 2024-12-11T06:16:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/77b79e559b2f48f9bcd18455f247f71b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b 2024-12-11T06:16:54,914 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b, entries=150, sequenceid=133, filesize=11.8 K 2024-12-11T06:16:54,915 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9c9700b86d470c316ec43a15e9141439 in 1262ms, sequenceid=133, compaction requested=false 2024-12-11T06:16:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-11T06:16:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-11T06:16:54,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-11T06:16:54,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-12-11T06:16:54,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4210 sec 2024-12-11T06:16:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:55,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733897813912, max timestamp=1733897815081Num uniques 859; 2024-12-11T06:16:55,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/4518703b2c1d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T06:16:55,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:55,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T06:16:55,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d449f95d59b44a9d9278b0a21e53aec1_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:55,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742370_1546 (size=12304) 2024-12-11T06:16:55,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897875132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897875239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:55,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897875449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:55,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d449f95d59b44a9d9278b0a21e53aec1_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d449f95d59b44a9d9278b0a21e53aec1_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:55,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7fa59c48f0594771a0381f7b0bdbd3dd, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:55,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7fa59c48f0594771a0381f7b0bdbd3dd is 175, key is test_row_0/A:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742371_1547 (size=31105) 2024-12-11T06:16:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-11T06:16:55,658 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:55,659 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-11T06:16:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:55,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:55,661 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:55,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897875759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:55,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:55,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:55,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:55,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:55,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7fa59c48f0594771a0381f7b0bdbd3dd 2024-12-11T06:16:55,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c0f7893f4e5542978bfdb5548eb654a6 is 50, key is test_row_0/B:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:55,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742372_1548 (size=12151) 2024-12-11T06:16:55,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:55,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:56,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:56,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37418 deadline: 1733897876238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:56,241 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:56,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897876269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:56,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:56,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c0f7893f4e5542978bfdb5548eb654a6 2024-12-11T06:16:56,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/0827233dd7774311bdf0c25a2caa04b5 is 50, key is test_row_0/C:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742373_1549 (size=12151) 2024-12-11T06:16:56,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897876369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:56,370 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897876398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:56,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897876398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:56,401 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:56,401 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:56,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:56,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/0827233dd7774311bdf0c25a2caa04b5 2024-12-11T06:16:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7fa59c48f0594771a0381f7b0bdbd3dd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd 2024-12-11T06:16:56,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd, entries=150, sequenceid=156, filesize=30.4 K 2024-12-11T06:16:56,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c0f7893f4e5542978bfdb5548eb654a6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6 2024-12-11T06:16:56,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6, entries=150, sequenceid=156, filesize=11.9 K 2024-12-11T06:16:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/0827233dd7774311bdf0c25a2caa04b5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5 2024-12-11T06:16:56,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5, entries=150, sequenceid=156, filesize=11.9 K 2024-12-11T06:16:56,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 9c9700b86d470c316ec43a15e9141439 in 1677ms, sequenceid=156, compaction requested=true 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:56,760 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:56,760 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:56,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:56,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:56,761 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:56,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:16:56,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=91.2 K 2024-12-11T06:16:56,761 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd] 2024-12-11T06:16:56,761 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43dd7ca6c0514b54ada27746a52d7b19, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e676174f0240fbad7b6406a662e81b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733897812208 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:16:56,762 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,762 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcfae8fc3d3f469788975d4092476bb9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.6 K 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fa59c48f0594771a0381f7b0bdbd3dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfae8fc3d3f469788975d4092476bb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 4024ce2f37e1462baa32b3e9f0c45582, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733897812208 2024-12-11T06:16:56,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f7893f4e5542978bfdb5548eb654a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:56,766 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:56,767 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:56,768 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/32a8415b44c944d3bf5fc331d8342a83 is 50, key is test_row_0/B:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:56,769 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121151d7e6e442944bf4adae879336b3f3a7_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:56,770 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121151d7e6e442944bf4adae879336b3f3a7_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:56,771 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121151d7e6e442944bf4adae879336b3f3a7_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742374_1550 (size=12493) 2024-12-11T06:16:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742375_1551 (size=4469) 2024-12-11T06:16:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:56,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-11T06:16:56,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:56,880 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:56,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:56,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a44aed47795a41ffa93771983212600e_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897815128/Put/seqid=0 2024-12-11T06:16:56,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742376_1552 (size=12304) 2024-12-11T06:16:56,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:56,891 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a44aed47795a41ffa93771983212600e_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a44aed47795a41ffa93771983212600e_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/104bc27943914010bb7eab448acdd569, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/104bc27943914010bb7eab448acdd569 is 175, key is test_row_0/A:col10/1733897815128/Put/seqid=0 2024-12-11T06:16:56,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742377_1553 (size=31105) 2024-12-11T06:16:56,896 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/104bc27943914010bb7eab448acdd569 2024-12-11T06:16:56,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/53268b53d2cb4d058b39f26a9476f017 is 50, key is test_row_0/B:col10/1733897815128/Put/seqid=0 2024-12-11T06:16:56,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742378_1554 (size=12151) 2024-12-11T06:16:57,180 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:57,181 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/3be122a86e9f4fb9b4fc3fe47a7ce797 is 175, key is test_row_0/A:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:57,195 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/32a8415b44c944d3bf5fc331d8342a83 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/32a8415b44c944d3bf5fc331d8342a83 2024-12-11T06:16:57,201 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into 32a8415b44c944d3bf5fc331d8342a83(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:57,201 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:57,201 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=13, startTime=1733897816760; duration=0sec 2024-12-11T06:16:57,201 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:57,201 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:16:57,201 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:57,202 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:57,202 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:16:57,202 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:57,203 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/7595bcfb6b68418fb58403ff0f163de2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.6 K 2024-12-11T06:16:57,203 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 7595bcfb6b68418fb58403ff0f163de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733897811705 2024-12-11T06:16:57,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b79e559b2f48f9bcd18455f247f71b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733897812208 2024-12-11T06:16:57,204 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 0827233dd7774311bdf0c25a2caa04b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:57,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742379_1555 (size=31447) 2024-12-11T06:16:57,213 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:57,213 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/2fdf178d369e4a1ebbc11c575c81154b is 50, key is test_row_0/C:col10/1733897813912/Put/seqid=0 2024-12-11T06:16:57,214 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/3be122a86e9f4fb9b4fc3fe47a7ce797 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797 2024-12-11T06:16:57,219 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into 3be122a86e9f4fb9b4fc3fe47a7ce797(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:57,219 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:57,219 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=13, startTime=1733897816760; duration=0sec 2024-12-11T06:16:57,219 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:57,219 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:16:57,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742380_1556 (size=12493) 2024-12-11T06:16:57,251 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/2fdf178d369e4a1ebbc11c575c81154b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/2fdf178d369e4a1ebbc11c575c81154b 2024-12-11T06:16:57,262 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into 2fdf178d369e4a1ebbc11c575c81154b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:57,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:57,262 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=13, startTime=1733897816760; duration=0sec 2024-12-11T06:16:57,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:57,262 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:16:57,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:57,305 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/53268b53d2cb4d058b39f26a9476f017 2024-12-11T06:16:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/d50be044062c4f13b03f386ef2859673 is 50, key is test_row_0/C:col10/1733897815128/Put/seqid=0 2024-12-11T06:16:57,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742381_1557 (size=12151) 2024-12-11T06:16:57,320 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/d50be044062c4f13b03f386ef2859673 2024-12-11T06:16:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/104bc27943914010bb7eab448acdd569 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569 2024-12-11T06:16:57,331 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569, entries=150, sequenceid=172, filesize=30.4 K 2024-12-11T06:16:57,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897877330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/53268b53d2cb4d058b39f26a9476f017 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017 2024-12-11T06:16:57,342 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017, entries=150, sequenceid=172, filesize=11.9 K 2024-12-11T06:16:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/d50be044062c4f13b03f386ef2859673 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673 2024-12-11T06:16:57,364 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673, entries=150, sequenceid=172, filesize=11.9 K 2024-12-11T06:16:57,366 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9c9700b86d470c316ec43a15e9141439 in 486ms, sequenceid=172, compaction requested=false 2024-12-11T06:16:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-11T06:16:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-11T06:16:57,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-11T06:16:57,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-12-11T06:16:57,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.7250 sec 2024-12-11T06:16:57,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:57,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:57,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211908df7da5a554cf9a14dfadb706f6841_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742382_1558 (size=14794) 2024-12-11T06:16:57,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:57,513 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211908df7da5a554cf9a14dfadb706f6841_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211908df7da5a554cf9a14dfadb706f6841_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:57,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/29391eaebb82470fbc87d091d285cbe3, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:57,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/29391eaebb82470fbc87d091d285cbe3 is 175, key is test_row_0/A:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:57,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742383_1559 (size=39749) 2024-12-11T06:16:57,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/29391eaebb82470fbc87d091d285cbe3 2024-12-11T06:16:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897877526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:57,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/b78a4d6d05df47d899872cec667efc04 is 50, key is test_row_0/B:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:57,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742384_1560 (size=12151) 2024-12-11T06:16:57,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/b78a4d6d05df47d899872cec667efc04 2024-12-11T06:16:57,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72e88d841aea49a28789c028ae45db11 is 50, key is test_row_0/C:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:57,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742385_1561 (size=12151) 2024-12-11T06:16:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897877639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-11T06:16:57,809 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:57,810 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-11T06:16:57,812 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T06:16:57,813 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:57,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897877849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T06:16:57,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T06:16:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:57,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:58,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72e88d841aea49a28789c028ae45db11 2024-12-11T06:16:58,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/29391eaebb82470fbc87d091d285cbe3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3 2024-12-11T06:16:58,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3, entries=200, sequenceid=192, filesize=38.8 K 2024-12-11T06:16:58,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/b78a4d6d05df47d899872cec667efc04 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04 2024-12-11T06:16:58,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04, entries=150, sequenceid=192, filesize=11.9 K 2024-12-11T06:16:58,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72e88d841aea49a28789c028ae45db11 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11 2024-12-11T06:16:58,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11, entries=150, sequenceid=192, filesize=11.9 K 2024-12-11T06:16:58,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9c9700b86d470c316ec43a15e9141439 in 583ms, sequenceid=192, compaction requested=true 2024-12-11T06:16:58,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:58,025 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:58,026 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:58,030 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:58,030 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:58,030 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:16:58,030 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:16:58,030 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,030 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,030 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=99.9 K 2024-12-11T06:16:58,030 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/32a8415b44c944d3bf5fc331d8342a83, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.9 K 2024-12-11T06:16:58,030 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,030 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3] 2024-12-11T06:16:58,031 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a8415b44c944d3bf5fc331d8342a83, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:58,031 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be122a86e9f4fb9b4fc3fe47a7ce797, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:58,032 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 53268b53d2cb4d058b39f26a9476f017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733897815123 2024-12-11T06:16:58,034 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b78a4d6d05df47d899872cec667efc04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817324 2024-12-11T06:16:58,037 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 104bc27943914010bb7eab448acdd569, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733897815123 2024-12-11T06:16:58,038 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29391eaebb82470fbc87d091d285cbe3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817312 2024-12-11T06:16:58,047 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:58,047 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/ee4cd4870f86434e8c17a637f3863e27 is 50, key is test_row_0/B:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:58,055 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:58,069 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121117343bf970fb46848cbe40f9634b9936_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:58,073 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121117343bf970fb46848cbe40f9634b9936_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:58,073 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121117343bf970fb46848cbe40f9634b9936_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:58,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742386_1562 (size=12595) 2024-12-11T06:16:58,102 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/ee4cd4870f86434e8c17a637f3863e27 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/ee4cd4870f86434e8c17a637f3863e27 2024-12-11T06:16:58,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742387_1563 (size=4469) 2024-12-11T06:16:58,110 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#475 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:58,110 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/c056007145f24d3195a3a1370255f901 is 175, key is test_row_0/A:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:58,114 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into ee4cd4870f86434e8c17a637f3863e27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:58,115 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:58,115 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=13, startTime=1733897818026; duration=0sec 2024-12-11T06:16:58,115 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:58,115 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:16:58,115 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:58,116 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:58,116 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:16:58,116 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,116 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/2fdf178d369e4a1ebbc11c575c81154b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=35.9 K 2024-12-11T06:16:58,117 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fdf178d369e4a1ebbc11c575c81154b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733897813912 2024-12-11T06:16:58,118 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting d50be044062c4f13b03f386ef2859673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733897815123 2024-12-11T06:16:58,118 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e88d841aea49a28789c028ae45db11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817324 2024-12-11T06:16:58,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T06:16:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,122 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T06:16:58,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bc55024d6c8b4f3ab20713293723853e_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897817515/Put/seqid=0 2024-12-11T06:16:58,158 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:58,159 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/66b7ebe42afc42f587981be83d3bc889 is 50, key is test_row_0/C:col10/1733897817324/Put/seqid=0 2024-12-11T06:16:58,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742388_1564 (size=31549) 2024-12-11T06:16:58,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:58,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742389_1565 (size=12304) 2024-12-11T06:16:58,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:58,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bc55024d6c8b4f3ab20713293723853e_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bc55024d6c8b4f3ab20713293723853e_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:58,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7ccf344277334d189d8af618f3f6d627, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:58,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7ccf344277334d189d8af618f3f6d627 is 175, key is test_row_0/A:col10/1733897817515/Put/seqid=0 2024-12-11T06:16:58,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742390_1566 (size=12595) 2024-12-11T06:16:58,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:58,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742391_1567 (size=31105) 2024-12-11T06:16:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897878237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897878353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T06:16:58,565 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/c056007145f24d3195a3a1370255f901 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901 2024-12-11T06:16:58,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897878568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:58,569 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into c056007145f24d3195a3a1370255f901(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:58,569 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:58,569 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=13, startTime=1733897818025; duration=0sec 2024-12-11T06:16:58,569 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:58,569 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:16:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:58,617 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/66b7ebe42afc42f587981be83d3bc889 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/66b7ebe42afc42f587981be83d3bc889 2024-12-11T06:16:58,620 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into 66b7ebe42afc42f587981be83d3bc889(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:58,620 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:58,620 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=13, startTime=1733897818026; duration=0sec 2024-12-11T06:16:58,620 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:58,620 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:16:58,643 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7ccf344277334d189d8af618f3f6d627 2024-12-11T06:16:58,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bcda70ed051d49348c62ce1ebfd4ce1d is 50, key is test_row_0/B:col10/1733897817515/Put/seqid=0 2024-12-11T06:16:58,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742392_1568 (size=12151) 2024-12-11T06:16:58,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bcda70ed051d49348c62ce1ebfd4ce1d 2024-12-11T06:16:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/85fcc3f124c0403c978a6b1f4f3dbed6 is 50, key is test_row_0/C:col10/1733897817515/Put/seqid=0 2024-12-11T06:16:58,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742393_1569 (size=12151) 2024-12-11T06:16:58,665 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/85fcc3f124c0403c978a6b1f4f3dbed6 2024-12-11T06:16:58,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/7ccf344277334d189d8af618f3f6d627 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627 2024-12-11T06:16:58,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627, entries=150, sequenceid=208, filesize=30.4 K 2024-12-11T06:16:58,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bcda70ed051d49348c62ce1ebfd4ce1d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d 2024-12-11T06:16:58,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d, entries=150, sequenceid=208, filesize=11.9 K 2024-12-11T06:16:58,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/85fcc3f124c0403c978a6b1f4f3dbed6 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6 2024-12-11T06:16:58,678 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6, entries=150, sequenceid=208, filesize=11.9 K 2024-12-11T06:16:58,679 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 9c9700b86d470c316ec43a15e9141439 in 557ms, sequenceid=208, compaction requested=false 2024-12-11T06:16:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:58,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-11T06:16:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-11T06:16:58,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-11T06:16:58,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 867 msec 2024-12-11T06:16:58,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 873 msec 2024-12-11T06:16:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:58,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:58,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113d19409959bf4226b99b5421ee1d1461_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:58,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742394_1570 (size=14794) 2024-12-11T06:16:58,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897878920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:58,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T06:16:58,948 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:16:58,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:16:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-11T06:16:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:16:58,950 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:16:58,950 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:16:58,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:16:59,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897879029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:16:59,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897879238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:59,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:16:59,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:16:59,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113d19409959bf4226b99b5421ee1d1461_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113d19409959bf4226b99b5421ee1d1461_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:16:59,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/0ee8871240914c4ea122bdc8bc71d19b, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:59,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/0ee8871240914c4ea122bdc8bc71d19b is 175, key is test_row_0/A:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:59,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742395_1571 (size=39749) 2024-12-11T06:16:59,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897879549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:16:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:16:59,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:16:59,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:59,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:16:59,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/0ee8871240914c4ea122bdc8bc71d19b 2024-12-11T06:16:59,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/9d33858ddd754e1295d4a4996ceec156 is 50, key is test_row_0/B:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:59,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:16:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T06:16:59,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742396_1572 (size=12151) 2024-12-11T06:16:59,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/9d33858ddd754e1295d4a4996ceec156 2024-12-11T06:16:59,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/bc9a27e832c14b5a85092b47b7f436cd is 50, key is test_row_0/C:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:59,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742397_1573 (size=12151) 2024-12-11T06:16:59,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/bc9a27e832c14b5a85092b47b7f436cd 2024-12-11T06:16:59,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/0ee8871240914c4ea122bdc8bc71d19b as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b 2024-12-11T06:16:59,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b, entries=200, sequenceid=228, filesize=38.8 K 2024-12-11T06:16:59,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/9d33858ddd754e1295d4a4996ceec156 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156 2024-12-11T06:16:59,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156, entries=150, sequenceid=228, filesize=11.9 K 2024-12-11T06:16:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/bc9a27e832c14b5a85092b47b7f436cd as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd 2024-12-11T06:16:59,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd, entries=150, sequenceid=228, filesize=11.9 K 2024-12-11T06:16:59,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9c9700b86d470c316ec43a15e9141439 in 876ms, sequenceid=228, compaction requested=true 2024-12-11T06:16:59,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:59,757 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:59,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:16:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:16:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:59,758 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:16:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:59,758 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:59,758 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:16:59,758 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,759 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=100.0 K 2024-12-11T06:16:59,759 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,759 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b] 2024-12-11T06:16:59,762 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c056007145f24d3195a3a1370255f901, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817324 2024-12-11T06:16:59,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:59,762 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:16:59,762 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,762 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/ee4cd4870f86434e8c17a637f3863e27, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=36.0 K 2024-12-11T06:16:59,763 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4cd4870f86434e8c17a637f3863e27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817324 2024-12-11T06:16:59,764 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ccf344277334d189d8af618f3f6d627, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733897817500 2024-12-11T06:16:59,764 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee8871240914c4ea122bdc8bc71d19b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:16:59,764 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bcda70ed051d49348c62ce1ebfd4ce1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733897817500 2024-12-11T06:16:59,764 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d33858ddd754e1295d4a4996ceec156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:16:59,771 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:59,772 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:59,772 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211bfb9bd48668e49db9513ab1a852fb6cc_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:59,772 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/b179bd45027e4a119559d3d91c994d9f is 50, key is test_row_0/B:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:59,774 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211bfb9bd48668e49db9513ab1a852fb6cc_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:59,774 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bfb9bd48668e49db9513ab1a852fb6cc_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:16:59,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742398_1574 (size=12697) 2024-12-11T06:16:59,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742399_1575 (size=4469) 2024-12-11T06:16:59,812 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/b179bd45027e4a119559d3d91c994d9f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b179bd45027e4a119559d3d91c994d9f 2024-12-11T06:16:59,818 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into b179bd45027e4a119559d3d91c994d9f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:59,818 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:59,818 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=13, startTime=1733897819758; duration=0sec 2024-12-11T06:16:59,818 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:16:59,818 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:16:59,818 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:16:59,819 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:16:59,819 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:16:59,820 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,820 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/66b7ebe42afc42f587981be83d3bc889, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=36.0 K 2024-12-11T06:16:59,820 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b7ebe42afc42f587981be83d3bc889, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733897817324 2024-12-11T06:16:59,820 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 85fcc3f124c0403c978a6b1f4f3dbed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733897817500 2024-12-11T06:16:59,820 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9a27e832c14b5a85092b47b7f436cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:16:59,826 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:16:59,827 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/c9ec1b33967a4f6b81c340933672f411 is 50, key is test_row_0/C:col10/1733897818222/Put/seqid=0 2024-12-11T06:16:59,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742400_1576 (size=12697) 2024-12-11T06:16:59,834 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/c9ec1b33967a4f6b81c340933672f411 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c9ec1b33967a4f6b81c340933672f411 2024-12-11T06:16:59,837 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into c9ec1b33967a4f6b81c340933672f411(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:16:59,837 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:16:59,837 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=13, startTime=1733897819758; duration=0sec 2024-12-11T06:16:59,837 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:16:59,837 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:16:59,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:16:59,865 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:16:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:16:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113589edb7ce894774808daf9e012e38e3_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897818919/Put/seqid=0 2024-12-11T06:16:59,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742401_1577 (size=12304) 2024-12-11T06:17:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:00,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:17:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:17:00,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897880113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,198 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:00,198 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f3d31079880b47b48a8e9adc3f97b876 is 175, key is test_row_0/A:col10/1733897818222/Put/seqid=0 2024-12-11T06:17:00,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742402_1578 (size=31651) 2024-12-11T06:17:00,206 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f3d31079880b47b48a8e9adc3f97b876 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876 2024-12-11T06:17:00,210 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into f3d31079880b47b48a8e9adc3f97b876(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:00,210 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:00,210 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=13, startTime=1733897819757; duration=0sec 2024-12-11T06:17:00,210 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:00,210 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:17:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897880219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:00,281 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412113589edb7ce894774808daf9e012e38e3_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113589edb7ce894774808daf9e012e38e3_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f894d3f6868c42528688657940712980, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f894d3f6868c42528688657940712980 is 175, key is test_row_0/A:col10/1733897818919/Put/seqid=0 2024-12-11T06:17:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742403_1579 (size=31105) 2024-12-11T06:17:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37450 deadline: 1733897880408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37392 deadline: 1733897880408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,410 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_0' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:17:00,410 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_2' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897880428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37470 deadline: 1733897880439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,441 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 4518703b2c1d,35425,1733897658824 for 'test_row_1' in 9c9700b86d470c316ec43a15e9141439 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-11T06:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f894d3f6868c42528688657940712980 2024-12-11T06:17:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1bf362c699564e2a840267f1ba77025c is 50, key is test_row_0/B:col10/1733897818919/Put/seqid=0 2024-12-11T06:17:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742404_1580 (size=12151) 2024-12-11T06:17:00,695 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1bf362c699564e2a840267f1ba77025c 2024-12-11T06:17:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e0bc7a47b741473bbc8664aca53e70b5 is 50, key is test_row_0/C:col10/1733897818919/Put/seqid=0 2024-12-11T06:17:00,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742405_1581 (size=12151) 2024-12-11T06:17:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897880739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:17:01,107 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e0bc7a47b741473bbc8664aca53e70b5 2024-12-11T06:17:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f894d3f6868c42528688657940712980 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980 2024-12-11T06:17:01,120 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980, entries=150, sequenceid=244, filesize=30.4 K 2024-12-11T06:17:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/1bf362c699564e2a840267f1ba77025c as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c 2024-12-11T06:17:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c, entries=150, sequenceid=244, filesize=11.9 K 2024-12-11T06:17:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e0bc7a47b741473bbc8664aca53e70b5 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5 2024-12-11T06:17:01,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5, entries=150, sequenceid=244, filesize=11.9 K 2024-12-11T06:17:01,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9c9700b86d470c316ec43a15e9141439 in 1266ms, sequenceid=244, compaction requested=false 2024-12-11T06:17:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:01,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-11T06:17:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-11T06:17:01,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-11T06:17:01,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-12-11T06:17:01,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1860 sec 2024-12-11T06:17:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:01,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:17:01,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:01,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211315841f3e89c48f8a4ea71c20e940201_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742406_1582 (size=12454) 2024-12-11T06:17:01,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:01,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211315841f3e89c48f8a4ea71c20e940201_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211315841f3e89c48f8a4ea71c20e940201_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:01,269 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/812912d4896a4261bc01efeeed6833af, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:01,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/812912d4896a4261bc01efeeed6833af is 175, key is test_row_0/A:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742407_1583 (size=31255) 2024-12-11T06:17:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897881282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897881388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:01,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897881599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:01,679 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/812912d4896a4261bc01efeeed6833af 2024-12-11T06:17:01,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c99091c9d3fa4d7b9e6d616431d066a2 is 50, key is test_row_0/B:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:01,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742408_1584 (size=12301) 2024-12-11T06:17:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897881911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:02,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c99091c9d3fa4d7b9e6d616431d066a2 2024-12-11T06:17:02,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/6fad0e861ce84c44a12b508c18a5a1ec is 50, key is test_row_0/C:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:02,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742409_1585 (size=12301) 2024-12-11T06:17:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897882418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:02,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:02,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/6fad0e861ce84c44a12b508c18a5a1ec 2024-12-11T06:17:02,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/812912d4896a4261bc01efeeed6833af as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af 2024-12-11T06:17:02,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af, entries=150, sequenceid=264, filesize=30.5 K 2024-12-11T06:17:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/c99091c9d3fa4d7b9e6d616431d066a2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2 2024-12-11T06:17:02,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2, entries=150, sequenceid=264, filesize=12.0 K 2024-12-11T06:17:02,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/6fad0e861ce84c44a12b508c18a5a1ec as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec 2024-12-11T06:17:02,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec, entries=150, sequenceid=264, filesize=12.0 K 2024-12-11T06:17:02,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9c9700b86d470c316ec43a15e9141439 in 1274ms, sequenceid=264, compaction requested=true 2024-12-11T06:17:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:17:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:02,526 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:17:02,527 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:17:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:17:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:17:02,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:17:02,527 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:17:02,528 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:17:02,528 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:02,528 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=91.8 K 2024-12-11T06:17:02,528 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:02,528 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af] 2024-12-11T06:17:02,528 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:17:02,528 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:17:02,528 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:02,528 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d31079880b47b48a8e9adc3f97b876, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:17:02,528 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b179bd45027e4a119559d3d91c994d9f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=36.3 K 2024-12-11T06:17:02,529 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting f894d3f6868c42528688657940712980, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733897818906 2024-12-11T06:17:02,529 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting b179bd45027e4a119559d3d91c994d9f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:17:02,529 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 812912d4896a4261bc01efeeed6833af, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:02,529 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf362c699564e2a840267f1ba77025c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733897818906 2024-12-11T06:17:02,529 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c99091c9d3fa4d7b9e6d616431d066a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:02,537 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:02,537 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/6a08bbf05b40489dacd5b71cd74f5099 is 50, key is test_row_0/B:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:02,545 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:02,560 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211d224c1bc12424c7d94534c08fa9c0220_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:02,562 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211d224c1bc12424c7d94534c08fa9c0220_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:02,562 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d224c1bc12424c7d94534c08fa9c0220_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:02,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742410_1586 (size=12949) 2024-12-11T06:17:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742411_1587 (size=4469) 2024-12-11T06:17:02,974 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/6a08bbf05b40489dacd5b71cd74f5099 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/6a08bbf05b40489dacd5b71cd74f5099 2024-12-11T06:17:02,978 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into 6a08bbf05b40489dacd5b71cd74f5099(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:02,978 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:02,978 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=13, startTime=1733897822526; duration=0sec 2024-12-11T06:17:02,978 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:17:02,978 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:17:02,978 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T06:17:02,979 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T06:17:02,979 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:17:02,979 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:02,979 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c9ec1b33967a4f6b81c340933672f411, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=36.3 K 2024-12-11T06:17:02,980 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ec1b33967a4f6b81c340933672f411, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733897818222 2024-12-11T06:17:02,980 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting e0bc7a47b741473bbc8664aca53e70b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733897818906 2024-12-11T06:17:02,980 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fad0e861ce84c44a12b508c18a5a1ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:02,985 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:02,985 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e3c01f1c93ab4148a199bf30ac0a7365 is 50, key is test_row_0/C:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:02,986 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:02,987 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/8037d50fb56a4cd691769b8a7dddcb39 is 175, key is test_row_0/A:col10/1733897820106/Put/seqid=0 2024-12-11T06:17:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742412_1588 (size=12949) 2024-12-11T06:17:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742413_1589 (size=31903) 2024-12-11T06:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T06:17:03,119 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:17:03,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-11T06:17:03,121 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:17:03,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:17:03,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T06:17:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T06:17:03,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:03,275 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:17:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:03,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d368a23781be44a9adf61d540495e112_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897821261/Put/seqid=0 2024-12-11T06:17:03,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742414_1590 (size=12454) 2024-12-11T06:17:03,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:03,289 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d368a23781be44a9adf61d540495e112_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d368a23781be44a9adf61d540495e112_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f03e70ef824d40ccbbd80007a62b05ea, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f03e70ef824d40ccbbd80007a62b05ea is 175, key is test_row_0/A:col10/1733897821261/Put/seqid=0 2024-12-11T06:17:03,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742415_1591 (size=31255) 2024-12-11T06:17:03,295 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f03e70ef824d40ccbbd80007a62b05ea 2024-12-11T06:17:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/feb3056e33c248d5ad998ef8c4de8799 is 50, key is test_row_0/B:col10/1733897821261/Put/seqid=0 2024-12-11T06:17:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742416_1592 (size=12301) 2024-12-11T06:17:03,406 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/8037d50fb56a4cd691769b8a7dddcb39 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39 2024-12-11T06:17:03,406 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/e3c01f1c93ab4148a199bf30ac0a7365 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e3c01f1c93ab4148a199bf30ac0a7365 2024-12-11T06:17:03,411 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into e3c01f1c93ab4148a199bf30ac0a7365(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:03,411 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:03,411 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=13, startTime=1733897822527; duration=0sec 2024-12-11T06:17:03,411 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:03,411 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:17:03,412 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into 8037d50fb56a4cd691769b8a7dddcb39(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:03,412 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:03,412 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=13, startTime=1733897822526; duration=0sec 2024-12-11T06:17:03,412 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:03,412 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:17:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:03,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:17:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T06:17:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897883494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897883608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:03,709 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/feb3056e33c248d5ad998ef8c4de8799 2024-12-11T06:17:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/c54ee9eb221d442fba8961a6ebace9ec is 50, key is test_row_0/C:col10/1733897821261/Put/seqid=0 2024-12-11T06:17:03,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742417_1593 (size=12301) 2024-12-11T06:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T06:17:03,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897883818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:04,126 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/c54ee9eb221d442fba8961a6ebace9ec 2024-12-11T06:17:04,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/f03e70ef824d40ccbbd80007a62b05ea as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea 2024-12-11T06:17:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897884128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:04,132 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea, entries=150, sequenceid=278, filesize=30.5 K 2024-12-11T06:17:04,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/feb3056e33c248d5ad998ef8c4de8799 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799 2024-12-11T06:17:04,136 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799, entries=150, sequenceid=278, filesize=12.0 K 2024-12-11T06:17:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/c54ee9eb221d442fba8961a6ebace9ec as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec 2024-12-11T06:17:04,139 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec, entries=150, sequenceid=278, filesize=12.0 K 2024-12-11T06:17:04,140 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9c9700b86d470c316ec43a15e9141439 in 865ms, sequenceid=278, compaction requested=false 2024-12-11T06:17:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-11T06:17:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-11T06:17:04,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-11T06:17:04,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-12-11T06:17:04,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0240 sec 2024-12-11T06:17:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T06:17:04,269 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:17:04,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-11T06:17:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-11T06:17:04,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T06:17:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:04,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T06:17:04,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T06:17:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:04,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35425 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:04,425 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:17:04,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:04,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211985cfa6c56e041038673dad90c8e5e2a_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897823493/Put/seqid=0 2024-12-11T06:17:04,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742418_1594 (size=12454) 2024-12-11T06:17:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:04,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. as already flushing 2024-12-11T06:17:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897884668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897884778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:04,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:04,841 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211985cfa6c56e041038673dad90c8e5e2a_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211985cfa6c56e041038673dad90c8e5e2a_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:04,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2a1c38d04be147d08bb8640dcd94e849, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2a1c38d04be147d08bb8640dcd94e849 is 175, key is test_row_0/A:col10/1733897823493/Put/seqid=0 2024-12-11T06:17:04,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742419_1595 (size=31255) 2024-12-11T06:17:04,845 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2a1c38d04be147d08bb8640dcd94e849 2024-12-11T06:17:04,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/143015ddd5334d0da57e9447c7bd27a1 is 50, key is test_row_0/B:col10/1733897823493/Put/seqid=0 2024-12-11T06:17:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742420_1596 (size=12301) 2024-12-11T06:17:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897884988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:05,177 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-12-11T06:17:05,177 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,177 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,177 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,177 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-12-11T06:17:05,177 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,177 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,177 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,179 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-12-11T06:17:05,179 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,179 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,179 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,179 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-12-11T06:17:05,180 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,180 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,180 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,180 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-12-11T06:17:05,180 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,180 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,180 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,262 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/143015ddd5334d0da57e9447c7bd27a1 2024-12-11T06:17:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72417ad5886d468698d6d2b483b35c49 is 50, key is test_row_0/C:col10/1733897823493/Put/seqid=0 2024-12-11T06:17:05,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742421_1597 (size=12301) 2024-12-11T06:17:05,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T06:17:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37428 deadline: 1733897885299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 , the old value is region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T06:17:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., hostname=4518703b2c1d,35425,1733897658824, seqNum=5 because the exception is null or not the one we care about 2024-12-11T06:17:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:05,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72417ad5886d468698d6d2b483b35c49 2024-12-11T06:17:05,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/2a1c38d04be147d08bb8640dcd94e849 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849 2024-12-11T06:17:05,676 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849, entries=150, sequenceid=299, filesize=30.5 K 2024-12-11T06:17:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/143015ddd5334d0da57e9447c7bd27a1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1 2024-12-11T06:17:05,680 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1, entries=150, sequenceid=299, filesize=12.0 K 2024-12-11T06:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/72417ad5886d468698d6d2b483b35c49 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49 2024-12-11T06:17:05,683 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49, entries=150, sequenceid=299, filesize=12.0 K 2024-12-11T06:17:05,684 INFO [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9c9700b86d470c316ec43a15e9141439 in 1259ms, sequenceid=299, compaction requested=true 2024-12-11T06:17:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4518703b2c1d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-11T06:17:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-11T06:17:05,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-11T06:17:05,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-12-11T06:17:05,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4170 sec 2024-12-11T06:17:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35425 {}] regionserver.HRegion(8855): Flush requested on 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:05,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:17:05,810 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:17:05,810 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:05,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:05,810 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,810 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:05,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:05,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211c111faf72d774fb385e01e76c775a77f_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742422_1598 (size=12454) 2024-12-11T06:17:06,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:06,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211c111faf72d774fb385e01e76c775a77f_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c111faf72d774fb385e01e76c775a77f_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:06,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/665e58c3eae146acb1bf51606fa46e3a, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:06,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/665e58c3eae146acb1bf51606fa46e3a is 175, key is test_row_0/A:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742423_1599 (size=31255) 2024-12-11T06:17:06,309 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-12-11T06:17:06,309 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:06,309 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:06,309 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:06,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T06:17:06,418 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:17:06,625 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/665e58c3eae146acb1bf51606fa46e3a 2024-12-11T06:17:06,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bb7568514859418a88e0c05d2dbd9a6e is 50, key is test_row_0/B:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:06,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742424_1600 (size=12301) 2024-12-11T06:17:07,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bb7568514859418a88e0c05d2dbd9a6e 2024-12-11T06:17:07,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ba83b78603d447c59ef1122a4d8dd77a is 50, key is test_row_0/C:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:07,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742425_1601 (size=12301) 2024-12-11T06:17:07,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ba83b78603d447c59ef1122a4d8dd77a 2024-12-11T06:17:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/665e58c3eae146acb1bf51606fa46e3a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a 2024-12-11T06:17:07,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a, entries=150, sequenceid=313, filesize=30.5 K 2024-12-11T06:17:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/bb7568514859418a88e0c05d2dbd9a6e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e 2024-12-11T06:17:07,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e, entries=150, sequenceid=313, filesize=12.0 K 2024-12-11T06:17:07,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/ba83b78603d447c59ef1122a4d8dd77a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a 2024-12-11T06:17:07,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a, entries=150, sequenceid=313, filesize=12.0 K 2024-12-11T06:17:07,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for 9c9700b86d470c316ec43a15e9141439 in 1644ms, sequenceid=313, compaction requested=true 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T06:17:07,453 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c9700b86d470c316ec43a15e9141439:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T06:17:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:17:07,453 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:17:07,454 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:17:07,454 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:17:07,454 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/B is initiating minor compaction (all files) 2024-12-11T06:17:07,454 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/A is initiating minor compaction (all files) 2024-12-11T06:17:07,454 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/B in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:07,454 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/A in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:07,454 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=122.7 K 2024-12-11T06:17:07,454 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/6a08bbf05b40489dacd5b71cd74f5099, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=48.7 K 2024-12-11T06:17:07,454 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:07,454 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. files: [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a] 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a08bbf05b40489dacd5b71cd74f5099, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 8037d50fb56a4cd691769b8a7dddcb39, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting f03e70ef824d40ccbbd80007a62b05ea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733897821261 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb3056e33c248d5ad998ef8c4de8799, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733897821261 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1c38d04be147d08bb8640dcd94e849, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733897823486 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143015ddd5334d0da57e9447c7bd27a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733897823486 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] compactions.Compactor(225): Compacting 665e58c3eae146acb1bf51606fa46e3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733897824660 2024-12-11T06:17:07,455 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7568514859418a88e0c05d2dbd9a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733897824660 2024-12-11T06:17:07,460 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:07,461 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#B#compaction#504 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:07,461 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3ef3adf5183746a4b9ba1e8ca4b5f5d2 is 50, key is test_row_0/B:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:07,462 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211f34a430ac64e44098f06e900a8090dd1_9c9700b86d470c316ec43a15e9141439 store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:07,470 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211f34a430ac64e44098f06e900a8090dd1_9c9700b86d470c316ec43a15e9141439, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:07,470 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f34a430ac64e44098f06e900a8090dd1_9c9700b86d470c316ec43a15e9141439 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:07,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742426_1602 (size=13085) 2024-12-11T06:17:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742427_1603 (size=4469) 2024-12-11T06:17:07,874 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/3ef3adf5183746a4b9ba1e8ca4b5f5d2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3ef3adf5183746a4b9ba1e8ca4b5f5d2 2024-12-11T06:17:07,874 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:07,875 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/25e63b6a653e43988b921a70c60756e2 is 175, key is test_row_0/A:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:07,877 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/B of 9c9700b86d470c316ec43a15e9141439 into 3ef3adf5183746a4b9ba1e8ca4b5f5d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:07,877 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:07,877 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/B, priority=12, startTime=1733897827453; duration=0sec 2024-12-11T06:17:07,878 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T06:17:07,878 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:B 2024-12-11T06:17:07,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742428_1604 (size=32039) 2024-12-11T06:17:07,878 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T06:17:07,878 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T06:17:07,879 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1541): 9c9700b86d470c316ec43a15e9141439/C is initiating minor compaction (all files) 2024-12-11T06:17:07,879 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c9700b86d470c316ec43a15e9141439/C in TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:07,879 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e3c01f1c93ab4148a199bf30ac0a7365, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a] into tmpdir=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp, totalSize=48.7 K 2024-12-11T06:17:07,879 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c01f1c93ab4148a199bf30ac0a7365, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733897820095 2024-12-11T06:17:07,879 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting c54ee9eb221d442fba8961a6ebace9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733897821261 2024-12-11T06:17:07,879 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72417ad5886d468698d6d2b483b35c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733897823486 2024-12-11T06:17:07,879 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba83b78603d447c59ef1122a4d8dd77a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733897824660 2024-12-11T06:17:07,885 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c9700b86d470c316ec43a15e9141439#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T06:17:07,886 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/1276c4d5153c462987a5f0812cb91d1e is 50, key is test_row_0/C:col10/1733897825808/Put/seqid=0 2024-12-11T06:17:07,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742429_1605 (size=13085) 2024-12-11T06:17:07,891 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/1276c4d5153c462987a5f0812cb91d1e as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/1276c4d5153c462987a5f0812cb91d1e 2024-12-11T06:17:07,894 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/C of 9c9700b86d470c316ec43a15e9141439 into 1276c4d5153c462987a5f0812cb91d1e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:07,894 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:07,894 INFO [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/C, priority=12, startTime=1733897827453; duration=0sec 2024-12-11T06:17:07,894 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:07,894 DEBUG [RS:0;4518703b2c1d:35425-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:C 2024-12-11T06:17:08,282 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/25e63b6a653e43988b921a70c60756e2 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/25e63b6a653e43988b921a70c60756e2 2024-12-11T06:17:08,286 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c9700b86d470c316ec43a15e9141439/A of 9c9700b86d470c316ec43a15e9141439 into 25e63b6a653e43988b921a70c60756e2(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T06:17:08,286 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c9700b86d470c316ec43a15e9141439: 2024-12-11T06:17:08,286 INFO [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., storeName=9c9700b86d470c316ec43a15e9141439/A, priority=12, startTime=1733897827453; duration=0sec 2024-12-11T06:17:08,286 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T06:17:08,286 DEBUG [RS:0;4518703b2c1d:35425-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c9700b86d470c316ec43a15e9141439:A 2024-12-11T06:17:10,430 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-12-11T06:17:10,430 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:10,430 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,430 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:10,459 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-12-11T06:17:10,459 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:10,459 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,459 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:10,510 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-12-11T06:17:10,510 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T06:17:10,510 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,510 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4118 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4062 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3902 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4134 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4060 2024-12-11T06:17:10,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T06:17:10,510 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:17:10,510 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:17:10,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:10,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:10,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-11T06:17:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:10,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897830514"}]},"ts":"1733897830514"} 2024-12-11T06:17:10,515 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T06:17:10,515 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T06:17:10,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T06:17:10,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, UNASSIGN}] 2024-12-11T06:17:10,517 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, UNASSIGN 2024-12-11T06:17:10,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=CLOSING, regionLocation=4518703b2c1d,35425,1733897658824 2024-12-11T06:17:10,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, UNASSIGN because future has completed 2024-12-11T06:17:10,519 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T06:17:10,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824}] 2024-12-11T06:17:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:10,671 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:10,671 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T06:17:10,671 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 9c9700b86d470c316ec43a15e9141439, disabling compactions & flushes 2024-12-11T06:17:10,671 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:10,671 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:10,671 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. after waiting 0 ms 2024-12-11T06:17:10,671 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:10,672 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 9c9700b86d470c316ec43a15e9141439 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=A 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=B 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c9700b86d470c316ec43a15e9141439, store=C 2024-12-11T06:17:10,672 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T06:17:10,676 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211261c6e8db04049de9a985a5f546f0f71_9c9700b86d470c316ec43a15e9141439 is 50, key is test_row_0/A:col10/1733897830429/Put/seqid=0 2024-12-11T06:17:10,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742430_1606 (size=12454) 2024-12-11T06:17:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:11,080 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T06:17:11,083 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211261c6e8db04049de9a985a5f546f0f71_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211261c6e8db04049de9a985a5f546f0f71_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:11,084 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/a9e78efa3a4c4bd08b82f025de94940d, store: [table=TestAcidGuarantees family=A region=9c9700b86d470c316ec43a15e9141439] 2024-12-11T06:17:11,084 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/a9e78efa3a4c4bd08b82f025de94940d is 175, key is test_row_0/A:col10/1733897830429/Put/seqid=0 2024-12-11T06:17:11,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742431_1607 (size=31255) 2024-12-11T06:17:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:11,488 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/a9e78efa3a4c4bd08b82f025de94940d 2024-12-11T06:17:11,553 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/728d2c9168c14782aa11d425626121fb is 50, key is test_row_0/B:col10/1733897830429/Put/seqid=0 2024-12-11T06:17:11,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742432_1608 (size=12301) 2024-12-11T06:17:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:11,956 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/728d2c9168c14782aa11d425626121fb 2024-12-11T06:17:11,962 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/61daed43067045519a6877201b3ccd58 is 50, key is test_row_0/C:col10/1733897830429/Put/seqid=0 2024-12-11T06:17:11,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742433_1609 (size=12301) 2024-12-11T06:17:11,965 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/61daed43067045519a6877201b3ccd58 2024-12-11T06:17:11,968 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/A/a9e78efa3a4c4bd08b82f025de94940d as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/a9e78efa3a4c4bd08b82f025de94940d 2024-12-11T06:17:11,970 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/a9e78efa3a4c4bd08b82f025de94940d, entries=150, sequenceid=323, filesize=30.5 K 2024-12-11T06:17:11,971 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/B/728d2c9168c14782aa11d425626121fb as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/728d2c9168c14782aa11d425626121fb 2024-12-11T06:17:11,973 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/728d2c9168c14782aa11d425626121fb, entries=150, sequenceid=323, filesize=12.0 K 2024-12-11T06:17:11,974 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/.tmp/C/61daed43067045519a6877201b3ccd58 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/61daed43067045519a6877201b3ccd58 2024-12-11T06:17:11,977 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/61daed43067045519a6877201b3ccd58, entries=150, sequenceid=323, filesize=12.0 K 2024-12-11T06:17:11,977 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9c9700b86d470c316ec43a15e9141439 in 1306ms, sequenceid=323, compaction requested=false 2024-12-11T06:17:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a] to archive 2024-12-11T06:17:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:17:11,980 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/754ad2173e224632892ebcc2194ecb0d 2024-12-11T06:17:11,980 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/4d57742c9ce54dbda006e5660accd9e7 2024-12-11T06:17:11,981 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/68220e35191c49dda49d0cf3e5968cad 2024-12-11T06:17:11,981 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/906af02e3d8247d38d418bb53b331b7f 2024-12-11T06:17:11,981 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2eddd57dcca44d46ac1d146d5e2d24b4 2024-12-11T06:17:11,981 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/5ef84b348bdd4c15b22973547bf5ee9e 2024-12-11T06:17:11,981 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/28e676174f0240fbad7b6406a662e81b 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/43dd7ca6c0514b54ada27746a52d7b19 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7fa59c48f0594771a0381f7b0bdbd3dd 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/3be122a86e9f4fb9b4fc3fe47a7ce797 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/38bb6405f6c44fd08bdf7c2243fb2001 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/104bc27943914010bb7eab448acdd569 2024-12-11T06:17:11,982 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/29391eaebb82470fbc87d091d285cbe3 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/c056007145f24d3195a3a1370255f901 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/7ccf344277334d189d8af618f3f6d627 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/0ee8871240914c4ea122bdc8bc71d19b 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f894d3f6868c42528688657940712980 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/8037d50fb56a4cd691769b8a7dddcb39 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f3d31079880b47b48a8e9adc3f97b876 2024-12-11T06:17:11,983 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/812912d4896a4261bc01efeeed6833af 2024-12-11T06:17:11,984 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/2a1c38d04be147d08bb8640dcd94e849 2024-12-11T06:17:11,984 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/f03e70ef824d40ccbbd80007a62b05ea 2024-12-11T06:17:11,984 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/665e58c3eae146acb1bf51606fa46e3a 2024-12-11T06:17:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/cc976353f990442faecbe73cf63e8a15, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcfae8fc3d3f469788975d4092476bb9, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/32a8415b44c944d3bf5fc331d8342a83, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/ee4cd4870f86434e8c17a637f3863e27, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b179bd45027e4a119559d3d91c994d9f, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/6a08bbf05b40489dacd5b71cd74f5099, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e] to archive 2024-12-11T06:17:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/34c3713a84864526a881a63fa799c1f3 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3a4db23f0cdd4707a48a8e34fcb767ff 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/cc976353f990442faecbe73cf63e8a15 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/cc976353f990442faecbe73cf63e8a15 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3b89980f33c9427ca2d2e9b9045bb459 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/dfd36b7286624950bd4c944ecf966ab9 2024-12-11T06:17:11,987 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/875a4ebeadd441f68984462ed6e877da 2024-12-11T06:17:11,988 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcfae8fc3d3f469788975d4092476bb9 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcfae8fc3d3f469788975d4092476bb9 2024-12-11T06:17:11,988 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1619464500ea4ad7ae8c36321c26ef22 2024-12-11T06:17:11,989 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/ee4cd4870f86434e8c17a637f3863e27 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/ee4cd4870f86434e8c17a637f3863e27 2024-12-11T06:17:11,989 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/32a8415b44c944d3bf5fc331d8342a83 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/32a8415b44c944d3bf5fc331d8342a83 2024-12-11T06:17:11,991 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/4024ce2f37e1462baa32b3e9f0c45582 2024-12-11T06:17:11,994 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b78a4d6d05df47d899872cec667efc04 2024-12-11T06:17:11,994 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c0f7893f4e5542978bfdb5548eb654a6 2024-12-11T06:17:11,995 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bcda70ed051d49348c62ce1ebfd4ce1d 2024-12-11T06:17:11,995 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/53268b53d2cb4d058b39f26a9476f017 2024-12-11T06:17:11,995 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b179bd45027e4a119559d3d91c994d9f to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/b179bd45027e4a119559d3d91c994d9f 2024-12-11T06:17:11,996 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/9d33858ddd754e1295d4a4996ceec156 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/6a08bbf05b40489dacd5b71cd74f5099 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/6a08bbf05b40489dacd5b71cd74f5099 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/1bf362c699564e2a840267f1ba77025c 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/bb7568514859418a88e0c05d2dbd9a6e 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/feb3056e33c248d5ad998ef8c4de8799 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/143015ddd5334d0da57e9447c7bd27a1 2024-12-11T06:17:11,997 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/c99091c9d3fa4d7b9e6d616431d066a2 2024-12-11T06:17:11,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0cc49ec7e63c49f980cf9ba884dcd2ca, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/7595bcfb6b68418fb58403ff0f163de2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/2fdf178d369e4a1ebbc11c575c81154b, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/66b7ebe42afc42f587981be83d3bc889, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c9ec1b33967a4f6b81c340933672f411, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e3c01f1c93ab4148a199bf30ac0a7365, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a] to archive 2024-12-11T06:17:11,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/027f6675312a4899bbd2c688a887e5d8 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0cc49ec7e63c49f980cf9ba884dcd2ca to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0cc49ec7e63c49f980cf9ba884dcd2ca 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ada5888d9dba441b9b521714523f4c4a 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/79cd5b7552924f369088ceecd48af2bf 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e8df996522864c51ad31bc91240f184c 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/f2631dd34b9640a2b7aa464be0cc2a37 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/7595bcfb6b68418fb58403ff0f163de2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/7595bcfb6b68418fb58403ff0f163de2 2024-12-11T06:17:12,001 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/8958869867a54c11b4ecee16a0a9d8f2 2024-12-11T06:17:12,002 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/2fdf178d369e4a1ebbc11c575c81154b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/2fdf178d369e4a1ebbc11c575c81154b 2024-12-11T06:17:12,002 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/0827233dd7774311bdf0c25a2caa04b5 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72e88d841aea49a28789c028ae45db11 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/77b79e559b2f48f9bcd18455f247f71b 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/85fcc3f124c0403c978a6b1f4f3dbed6 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/66b7ebe42afc42f587981be83d3bc889 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/66b7ebe42afc42f587981be83d3bc889 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/d50be044062c4f13b03f386ef2859673 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c9ec1b33967a4f6b81c340933672f411 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c9ec1b33967a4f6b81c340933672f411 2024-12-11T06:17:12,003 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/bc9a27e832c14b5a85092b47b7f436cd 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e0bc7a47b741473bbc8664aca53e70b5 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/6fad0e861ce84c44a12b508c18a5a1ec 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/c54ee9eb221d442fba8961a6ebace9ec 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e3c01f1c93ab4148a199bf30ac0a7365 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/e3c01f1c93ab4148a199bf30ac0a7365 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/72417ad5886d468698d6d2b483b35c49 2024-12-11T06:17:12,004 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/ba83b78603d447c59ef1122a4d8dd77a 2024-12-11T06:17:12,007 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-12-11T06:17:12,008 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. 2024-12-11T06:17:12,008 DEBUG [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 9c9700b86d470c316ec43a15e9141439: Waiting for close lock at 1733897830671Running coprocessor pre-close hooks at 1733897830671Disabling compacts and flushes for region at 1733897830671Disabling writes for close at 1733897830671Obtaining lock to block concurrent updates at 1733897830672 (+1 ms)Preparing flush snapshotting stores in 9c9700b86d470c316ec43a15e9141439 at 1733897830672Finished memstore snapshotting TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733897830672Flushing stores of TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439. at 1733897830672Flushing 9c9700b86d470c316ec43a15e9141439/A: creating writer at 1733897830672Flushing mob file 9c9700b86d470c316ec43a15e9141439/A: appending metadata at 1733897830676 (+4 ms)Flushing mob file 9c9700b86d470c316ec43a15e9141439/A: closing flushed file at 1733897830676Flushing 9c9700b86d470c316ec43a15e9141439/A: appending metadata at 1733897831084 (+408 ms)Flushing 9c9700b86d470c316ec43a15e9141439/A: closing flushed file at 1733897831084Flushing 9c9700b86d470c316ec43a15e9141439/B: creating writer at 1733897831550 (+466 ms)Flushing 9c9700b86d470c316ec43a15e9141439/B: appending metadata at 1733897831552 (+2 ms)Flushing 9c9700b86d470c316ec43a15e9141439/B: closing flushed file at 1733897831552Flushing 9c9700b86d470c316ec43a15e9141439/C: creating writer at 1733897831960 (+408 ms)Flushing 9c9700b86d470c316ec43a15e9141439/C: appending metadata at 1733897831961 (+1 ms)Flushing 9c9700b86d470c316ec43a15e9141439/C: closing flushed file at 1733897831962 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e6be45: reopening flushed file at 1733897831967 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@284caf49: reopening flushed file at 1733897831970 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@427be441: reopening flushed file at 1733897831974 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9c9700b86d470c316ec43a15e9141439 in 1306ms, sequenceid=323, compaction requested=false at 1733897831977 (+3 ms)Writing region close event to WAL at 1733897832005 (+28 ms)Running coprocessor post-close hooks at 1733897832008 (+3 ms)Closed at 1733897832008 2024-12-11T06:17:12,009 INFO [RS_CLOSE_REGION-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,010 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=9c9700b86d470c316ec43a15e9141439, regionState=CLOSED 2024-12-11T06:17:12,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 because future has completed 2024-12-11T06:17:12,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-11T06:17:12,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c9700b86d470c316ec43a15e9141439, server=4518703b2c1d,35425,1733897658824 in 1.4930 sec 2024-12-11T06:17:12,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-11T06:17:12,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c9700b86d470c316ec43a15e9141439, UNASSIGN in 1.4960 sec 2024-12-11T06:17:12,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-11T06:17:12,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-12-11T06:17:12,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733897832018"}]},"ts":"1733897832018"} 2024-12-11T06:17:12,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T06:17:12,019 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T06:17:12,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5090 sec 2024-12-11T06:17:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T06:17:12,659 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:17:12,660 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-11T06:17:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T06:17:12,664 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,666 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C, FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits] 2024-12-11T06:17:12,667 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/25e63b6a653e43988b921a70c60756e2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/25e63b6a653e43988b921a70c60756e2 2024-12-11T06:17:12,668 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/a9e78efa3a4c4bd08b82f025de94940d to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/A/a9e78efa3a4c4bd08b82f025de94940d 2024-12-11T06:17:12,669 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3ef3adf5183746a4b9ba1e8ca4b5f5d2 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/3ef3adf5183746a4b9ba1e8ca4b5f5d2 2024-12-11T06:17:12,669 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/728d2c9168c14782aa11d425626121fb to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/B/728d2c9168c14782aa11d425626121fb 2024-12-11T06:17:12,671 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/1276c4d5153c462987a5f0812cb91d1e to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/1276c4d5153c462987a5f0812cb91d1e 2024-12-11T06:17:12,671 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/61daed43067045519a6877201b3ccd58 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/C/61daed43067045519a6877201b3ccd58 2024-12-11T06:17:12,673 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits/326.seqid to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439/recovered.edits/326.seqid 2024-12-11T06:17:12,673 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/default/TestAcidGuarantees/9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,673 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T06:17:12,673 DEBUG [PEWorker-2 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:17:12,674 DEBUG [PEWorker-2 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T06:17:12,678 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211261c6e8db04049de9a985a5f546f0f71_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211261c6e8db04049de9a985a5f546f0f71_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,678 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211315841f3e89c48f8a4ea71c20e940201_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211315841f3e89c48f8a4ea71c20e940201_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,678 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113589edb7ce894774808daf9e012e38e3_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113589edb7ce894774808daf9e012e38e3_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,678 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113b65840d0c894e09b10a80aef174ba5f_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113b65840d0c894e09b10a80aef174ba5f_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,678 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121148c3a932f7d14fe4b815dd7ccf3ad9a4_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121148c3a932f7d14fe4b815dd7ccf3ad9a4_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,679 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113d19409959bf4226b99b5421ee1d1461_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412113d19409959bf4226b99b5421ee1d1461_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,679 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412116b5d51681ef34252a6ed651334cbc7b7_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412116b5d51681ef34252a6ed651334cbc7b7_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,679 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114926e508af304735ad2400757d4c17ab_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114926e508af304735ad2400757d4c17ab_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,679 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211908df7da5a554cf9a14dfadb706f6841_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211908df7da5a554cf9a14dfadb706f6841_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,679 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121174d64ed3fd80476fbec8082720d809a5_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121174d64ed3fd80476fbec8082720d809a5_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211985cfa6c56e041038673dad90c8e5e2a_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211985cfa6c56e041038673dad90c8e5e2a_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bc55024d6c8b4f3ab20713293723853e_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bc55024d6c8b4f3ab20713293723853e_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a44aed47795a41ffa93771983212600e_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a44aed47795a41ffa93771983212600e_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c111faf72d774fb385e01e76c775a77f_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211c111faf72d774fb385e01e76c775a77f_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d449f95d59b44a9d9278b0a21e53aec1_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d449f95d59b44a9d9278b0a21e53aec1_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f16b305b982045399fcc55b7adb77db3_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f16b305b982045399fcc55b7adb77db3_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,680 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d368a23781be44a9adf61d540495e112_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d368a23781be44a9adf61d540495e112_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,681 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f5b967678cfb484896847b6750b19ab7_9c9700b86d470c316ec43a15e9141439 to hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f5b967678cfb484896847b6750b19ab7_9c9700b86d470c316ec43a15e9141439 2024-12-11T06:17:12,681 DEBUG [PEWorker-2 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T06:17:12,683 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,685 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T06:17:12,686 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T06:17:12,687 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,687 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T06:17:12,687 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733897832687"}]},"ts":"9223372036854775807"} 2024-12-11T06:17:12,689 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T06:17:12,689 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9c9700b86d470c316ec43a15e9141439, NAME => 'TestAcidGuarantees,,1733897802038.9c9700b86d470c316ec43a15e9141439.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T06:17:12,689 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T06:17:12,689 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733897832689"}]},"ts":"9223372036854775807"} 2024-12-11T06:17:12,691 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T06:17:12,691 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T06:17:12,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-11T06:17:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T06:17:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T06:17:12,770 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T06:17:12,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255 (was 255), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=750 (was 837), ProcessCount=11 (was 11), AvailableMemoryMB=3009 (was 238) - AvailableMemoryMB LEAK? - 2024-12-11T06:17:12,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T06:17:12,782 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:17:12,782 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:17:12,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:12,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:12,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T06:17:12,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T06:17:12,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1866486918, stopped=false 2024-12-11T06:17:12,784 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4518703b2c1d,41397,1733897657727 2024-12-11T06:17:12,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T06:17:12,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T06:17:12,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:17:12,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:17:12,834 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T06:17:12,834 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T06:17:12,835 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:17:12,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:12,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T06:17:12,836 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4518703b2c1d,35425,1733897658824' ***** 2024-12-11T06:17:12,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T06:17:12,837 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T06:17:12,838 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T06:17:12,838 INFO [RS:0;4518703b2c1d:35425 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T06:17:12,838 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T06:17:12,838 INFO [RS:0;4518703b2c1d:35425 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(959): stopping server 4518703b2c1d,35425,1733897658824 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4518703b2c1d:35425. 2024-12-11T06:17:12,839 DEBUG [RS:0;4518703b2c1d:35425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T06:17:12,839 DEBUG [RS:0;4518703b2c1d:35425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T06:17:12,839 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T06:17:12,840 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-11T06:17:12,840 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-11T06:17:12,840 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:12,840 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T06:17:12,840 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T06:17:12,840 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T06:17:12,840 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T06:17:12,840 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T06:17:12,841 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-11T06:17:12,860 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/info/f1993af6990a4ab08560cd815eb162e3 is 95, key is TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5./info:/1733897696805/DeleteFamily/seqid=0 2024-12-11T06:17:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742434_1610 (size=6514) 2024-12-11T06:17:12,884 INFO [regionserver/4518703b2c1d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T06:17:13,041 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:13,241 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:13,264 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/info/f1993af6990a4ab08560cd815eb162e3 2024-12-11T06:17:13,280 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/ns/ce604dbddeea407592e183650b0ba284 is 93, key is TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5./ns:/1733897696743/DeleteFamily/seqid=0 2024-12-11T06:17:13,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742435_1611 (size=5906) 2024-12-11T06:17:13,441 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:13,641 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:13,683 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/ns/ce604dbddeea407592e183650b0ba284 2024-12-11T06:17:13,700 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/rep_barrier/6a638f7bb5cd408a8c9d6a6e5503e724 is 102, key is TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5./rep_barrier:/1733897696743/DeleteFamily/seqid=0 2024-12-11T06:17:13,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742436_1612 (size=6025) 2024-12-11T06:17:13,764 INFO [regionserver/4518703b2c1d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T06:17:13,764 INFO [regionserver/4518703b2c1d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T06:17:13,841 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:14,042 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:14,103 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/rep_barrier/6a638f7bb5cd408a8c9d6a6e5503e724 2024-12-11T06:17:14,118 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/table/8610032396804870ad78a346bc19db3a is 96, key is TestAcidGuarantees,,1733897663566.54b3d700051ccb5e9981f4a99fd0aae5./table:/1733897696743/DeleteFamily/seqid=0 2024-12-11T06:17:14,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742437_1613 (size=5936) 2024-12-11T06:17:14,242 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:14,442 DEBUG [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T06:17:14,522 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/table/8610032396804870ad78a346bc19db3a 2024-12-11T06:17:14,526 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/info/f1993af6990a4ab08560cd815eb162e3 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/info/f1993af6990a4ab08560cd815eb162e3 2024-12-11T06:17:14,530 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/info/f1993af6990a4ab08560cd815eb162e3, entries=12, sequenceid=90, filesize=6.4 K 2024-12-11T06:17:14,531 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/ns/ce604dbddeea407592e183650b0ba284 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/ns/ce604dbddeea407592e183650b0ba284 2024-12-11T06:17:14,535 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/ns/ce604dbddeea407592e183650b0ba284, entries=8, sequenceid=90, filesize=5.8 K 2024-12-11T06:17:14,536 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/rep_barrier/6a638f7bb5cd408a8c9d6a6e5503e724 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/rep_barrier/6a638f7bb5cd408a8c9d6a6e5503e724 2024-12-11T06:17:14,539 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/rep_barrier/6a638f7bb5cd408a8c9d6a6e5503e724, entries=6, sequenceid=90, filesize=5.9 K 2024-12-11T06:17:14,540 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/.tmp/table/8610032396804870ad78a346bc19db3a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/table/8610032396804870ad78a346bc19db3a 2024-12-11T06:17:14,543 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/table/8610032396804870ad78a346bc19db3a, entries=7, sequenceid=90, filesize=5.8 K 2024-12-11T06:17:14,544 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-12-11T06:17:14,549 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-11T06:17:14,550 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T06:17:14,550 INFO [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T06:17:14,550 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733897832840Running coprocessor pre-close hooks at 1733897832840Disabling compacts and flushes for region at 1733897832840Disabling writes for close at 1733897832840Obtaining lock to block concurrent updates at 1733897832841 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733897832841Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733897832841Flushing stores of hbase:meta,,1.1588230740 at 1733897832842 (+1 ms)Flushing 1588230740/info: creating writer at 1733897832842Flushing 1588230740/info: appending metadata at 1733897832860 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733897832860Flushing 1588230740/ns: creating writer at 1733897833267 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733897833279 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733897833279Flushing 1588230740/rep_barrier: creating writer at 1733897833687 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733897833699 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733897833700 (+1 ms)Flushing 1588230740/table: creating writer at 1733897834106 (+406 ms)Flushing 1588230740/table: appending metadata at 1733897834118 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733897834118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ecda099: reopening flushed file at 1733897834525 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a76c12e: reopening flushed file at 1733897834530 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f74b2b8: reopening flushed file at 1733897834535 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@291cb945: reopening flushed file at 1733897834539 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1733897834544 (+5 ms)Writing region close event to WAL at 1733897834546 (+2 ms)Running coprocessor post-close hooks at 1733897834550 (+4 ms)Closed at 1733897834550 2024-12-11T06:17:14,550 DEBUG [RS_CLOSE_META-regionserver/4518703b2c1d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T06:17:14,643 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(976): stopping server 4518703b2c1d,35425,1733897658824; all regions closed. 2024-12-11T06:17:14,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741834_1010 (size=25344) 2024-12-11T06:17:14,701 DEBUG [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/oldWALs 2024-12-11T06:17:14,701 INFO [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4518703b2c1d%2C35425%2C1733897658824.meta:.meta(num 1733897662949) 2024-12-11T06:17:14,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741833_1009 (size=21614164) 2024-12-11T06:17:14,707 DEBUG [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/oldWALs 2024-12-11T06:17:14,707 INFO [RS:0;4518703b2c1d:35425 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4518703b2c1d%2C35425%2C1733897658824:(num 1733897661939) 2024-12-11T06:17:14,707 DEBUG [RS:0;4518703b2c1d:35425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T06:17:14,707 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T06:17:14,707 INFO [RS:0;4518703b2c1d:35425 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T06:17:14,707 INFO [RS:0;4518703b2c1d:35425 {}] hbase.ChoreService(370): Chore service for: regionserver/4518703b2c1d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T06:17:14,707 INFO [RS:0;4518703b2c1d:35425 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T06:17:14,707 INFO [regionserver/4518703b2c1d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T06:17:14,708 INFO [RS:0;4518703b2c1d:35425 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35425 2024-12-11T06:17:14,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4518703b2c1d,35425,1733897658824 2024-12-11T06:17:14,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T06:17:14,797 INFO [RS:0;4518703b2c1d:35425 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T06:17:14,798 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4518703b2c1d,35425,1733897658824] 2024-12-11T06:17:15,017 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4518703b2c1d,35425,1733897658824 already deleted, retry=false 2024-12-11T06:17:15,017 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4518703b2c1d,35425,1733897658824 expired; onlineServers=0 2024-12-11T06:17:15,017 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4518703b2c1d,41397,1733897657727' ***** 2024-12-11T06:17:15,018 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T06:17:15,018 INFO [M:0;4518703b2c1d:41397 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T06:17:15,018 INFO [M:0;4518703b2c1d:41397 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T06:17:15,018 DEBUG [M:0;4518703b2c1d:41397 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T06:17:15,018 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T06:17:15,018 DEBUG [M:0;4518703b2c1d:41397 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T06:17:15,018 DEBUG [master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.small.0-1733897661535 {}] cleaner.HFileCleaner(306): Exit Thread[master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.small.0-1733897661535,5,FailOnTimeoutGroup] 2024-12-11T06:17:15,018 DEBUG [master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.large.0-1733897661533 {}] cleaner.HFileCleaner(306): Exit Thread[master/4518703b2c1d:0:becomeActiveMaster-HFileCleaner.large.0-1733897661533,5,FailOnTimeoutGroup] 2024-12-11T06:17:15,019 INFO [M:0;4518703b2c1d:41397 {}] hbase.ChoreService(370): Chore service for: master/4518703b2c1d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T06:17:15,019 INFO [M:0;4518703b2c1d:41397 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T06:17:15,019 DEBUG [M:0;4518703b2c1d:41397 {}] master.HMaster(1795): Stopping service threads 2024-12-11T06:17:15,019 INFO [M:0;4518703b2c1d:41397 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T06:17:15,019 INFO [M:0;4518703b2c1d:41397 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T06:17:15,019 ERROR [M:0;4518703b2c1d:41397 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:33311 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:33311,5,PEWorkerGroup] 2024-12-11T06:17:15,020 INFO [M:0;4518703b2c1d:41397 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T06:17:15,021 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T06:17:15,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T06:17:15,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35425-0x1001382a0340001, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T06:17:15,054 INFO [RS:0;4518703b2c1d:35425 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T06:17:15,054 INFO [RS:0;4518703b2c1d:35425 {}] regionserver.HRegionServer(1031): Exiting; stopping=4518703b2c1d,35425,1733897658824; zookeeper connection closed. 2024-12-11T06:17:15,055 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@569f3cf3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@569f3cf3 2024-12-11T06:17:15,055 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-11T06:17:15,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T06:17:15,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T06:17:15,115 DEBUG [M:0;4518703b2c1d:41397 {}] zookeeper.ZKUtil(347): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T06:17:15,115 WARN [M:0;4518703b2c1d:41397 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T06:17:15,115 INFO [M:0;4518703b2c1d:41397 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/.lastflushedseqids 2024-12-11T06:17:15,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742438_1614 (size=72) 2024-12-11T06:17:15,528 INFO [M:0;4518703b2c1d:41397 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T06:17:15,528 INFO [M:0;4518703b2c1d:41397 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T06:17:15,528 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T06:17:15,528 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:17:15,528 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:17:15,528 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T06:17:15,528 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:17:15,528 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=751.36 KB heapSize=918.83 KB 2024-12-11T06:17:15,548 DEBUG [M:0;4518703b2c1d:41397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36841e9677174e89bc4cea4c53a254b1 is 82, key is hbase:meta,,1/info:regioninfo/1733897663050/Put/seqid=0 2024-12-11T06:17:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742439_1615 (size=5672) 2024-12-11T06:17:15,554 INFO [M:0;4518703b2c1d:41397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2063 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36841e9677174e89bc4cea4c53a254b1 2024-12-11T06:17:15,581 DEBUG [M:0;4518703b2c1d:41397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/885b06dd10a64a798918a09ba27f749a is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x1F/proc:d/1733897699226/Put/seqid=0 2024-12-11T06:17:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742440_1616 (size=43967) 2024-12-11T06:17:15,991 INFO [M:0;4518703b2c1d:41397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.81 KB at sequenceid=2063 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/885b06dd10a64a798918a09ba27f749a 2024-12-11T06:17:16,000 INFO [M:0;4518703b2c1d:41397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 885b06dd10a64a798918a09ba27f749a 2024-12-11T06:17:16,017 DEBUG [M:0;4518703b2c1d:41397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54cc48ad6d64409981f3ba12f441232f is 69, key is 4518703b2c1d,35425,1733897658824/rs:state/1733897661589/Put/seqid=0 2024-12-11T06:17:16,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073742441_1617 (size=5156) 2024-12-11T06:17:16,421 INFO [M:0;4518703b2c1d:41397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2063 (bloomFilter=true), to=hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54cc48ad6d64409981f3ba12f441232f 2024-12-11T06:17:16,424 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36841e9677174e89bc4cea4c53a254b1 as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/36841e9677174e89bc4cea4c53a254b1 2024-12-11T06:17:16,426 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/36841e9677174e89bc4cea4c53a254b1, entries=8, sequenceid=2063, filesize=5.5 K 2024-12-11T06:17:16,427 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/885b06dd10a64a798918a09ba27f749a as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/885b06dd10a64a798918a09ba27f749a 2024-12-11T06:17:16,430 INFO [M:0;4518703b2c1d:41397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 885b06dd10a64a798918a09ba27f749a 2024-12-11T06:17:16,430 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/885b06dd10a64a798918a09ba27f749a, entries=162, sequenceid=2063, filesize=42.9 K 2024-12-11T06:17:16,430 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54cc48ad6d64409981f3ba12f441232f as hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/54cc48ad6d64409981f3ba12f441232f 2024-12-11T06:17:16,433 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33311/user/jenkins/test-data/2f9351ca-600d-0c1d-c817-29c061f08915/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/54cc48ad6d64409981f3ba12f441232f, entries=1, sequenceid=2063, filesize=5.0 K 2024-12-11T06:17:16,433 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(3140): Finished flush of dataSize ~751.36 KB/769395, heapSize ~918.53 KB/940576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 905ms, sequenceid=2063, compaction requested=false 2024-12-11T06:17:16,435 INFO [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T06:17:16,435 DEBUG [M:0;4518703b2c1d:41397 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733897835528Disabling compacts and flushes for region at 1733897835528Disabling writes for close at 1733897835528Obtaining lock to block concurrent updates at 1733897835528Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733897835528Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=769395, getHeapSize=940816, getOffHeapSize=0, getCellsCount=2250 at 1733897835529 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733897835529Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733897835529Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733897835548 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733897835548Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733897835558 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733897835580 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733897835580Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733897836001 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733897836017 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733897836017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a3bf922: reopening flushed file at 1733897836423 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c28d80e: reopening flushed file at 1733897836427 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2022ba: reopening flushed file at 1733897836430 (+3 ms)Finished flush of dataSize ~751.36 KB/769395, heapSize ~918.53 KB/940576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 905ms, sequenceid=2063, compaction requested=false at 1733897836433 (+3 ms)Writing region close event to WAL at 1733897836435 (+2 ms)Closed at 1733897836435 2024-12-11T06:17:16,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33105 is added to blk_1073741830_1006 (size=904657) 2024-12-11T06:17:16,438 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T06:17:16,438 INFO [M:0;4518703b2c1d:41397 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T06:17:16,438 INFO [M:0;4518703b2c1d:41397 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41397 2024-12-11T06:17:16,438 INFO [M:0;4518703b2c1d:41397 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T06:17:16,565 INFO [M:0;4518703b2c1d:41397 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T06:17:16,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T06:17:16,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41397-0x1001382a0340000, quorum=127.0.0.1:60143, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T06:17:16,571 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T06:17:16,574 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T06:17:16,574 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T06:17:16,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T06:17:16,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.log.dir/,STOPPED} 2024-12-11T06:17:16,578 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T06:17:16,578 WARN [BP-461083816-172.17.0.3-1733897652939 heartbeating to localhost/127.0.0.1:33311 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T06:17:16,579 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T06:17:16,579 WARN [BP-461083816-172.17.0.3-1733897652939 heartbeating to localhost/127.0.0.1:33311 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-461083816-172.17.0.3-1733897652939 (Datanode Uuid 24e1446c-b24e-40c8-80d4-824be2c4b671) service to localhost/127.0.0.1:33311 2024-12-11T06:17:16,580 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data1/current/BP-461083816-172.17.0.3-1733897652939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T06:17:16,581 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/cluster_62c446f8-3327-0fd5-9ea9-839dbc106920/data/data2/current/BP-461083816-172.17.0.3-1733897652939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T06:17:16,581 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T06:17:16,593 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T06:17:16,594 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T06:17:16,594 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T06:17:16,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T06:17:16,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c988e48a-f188-ba7b-b50e-ef0a6453fd11/hadoop.log.dir/,STOPPED} 2024-12-11T06:17:16,631 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T06:17:16,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down